From 993595df73b835ddff115ddae67ab3d9108116a2 Mon Sep 17 00:00:00 2001 From: wwqgtxx Date: Sat, 17 Jan 2026 13:49:34 +0800 Subject: [PATCH] chore: switch to our own common/orderedmap package, remove two unneeded json dependence --- common/orderedmap/doc.go | 8 + common/orderedmap/json.go | 139 ++++++++++ common/orderedmap/json_fuzz_test.go | 117 ++++++++ common/orderedmap/json_test.go | 338 +++++++++++++++++++++++ common/orderedmap/orderedmap.go | 295 ++++++++++++++++++++ common/orderedmap/orderedmap_test.go | 384 +++++++++++++++++++++++++++ common/orderedmap/utils_test.go | 76 ++++++ common/orderedmap/yaml.go | 71 +++++ common/orderedmap/yaml_fuzz_test.go | 82 ++++++ common/orderedmap/yaml_test.go | 334 +++++++++++++++++++++++ config/config.go | 2 +- go.mod | 3 - go.sum | 7 - 13 files changed, 1845 insertions(+), 11 deletions(-) create mode 100644 common/orderedmap/doc.go create mode 100644 common/orderedmap/json.go create mode 100644 common/orderedmap/json_fuzz_test.go create mode 100644 common/orderedmap/json_test.go create mode 100644 common/orderedmap/orderedmap.go create mode 100644 common/orderedmap/orderedmap_test.go create mode 100644 common/orderedmap/utils_test.go create mode 100644 common/orderedmap/yaml.go create mode 100644 common/orderedmap/yaml_fuzz_test.go create mode 100644 common/orderedmap/yaml_test.go diff --git a/common/orderedmap/doc.go b/common/orderedmap/doc.go new file mode 100644 index 00000000..f8ca4156 --- /dev/null +++ b/common/orderedmap/doc.go @@ -0,0 +1,8 @@ +package orderedmap + +// copy and modified from https://github.com/wk8/go-ordered-map/tree/v2.1.8 +// which is licensed under Apache v2. +// +// mihomo modified: +// 1. remove dependence of mailru/easyjson for MarshalJSON +// 2. remove dependence of buger/jsonparser for UnmarshalJSON diff --git a/common/orderedmap/json.go b/common/orderedmap/json.go new file mode 100644 index 00000000..c1800212 --- /dev/null +++ b/common/orderedmap/json.go @@ -0,0 +1,139 @@ +package orderedmap + +import ( + "bytes" + "encoding" + "encoding/json" + "errors" + "fmt" + "reflect" +) + +var ( + _ json.Marshaler = &OrderedMap[int, any]{} + _ json.Unmarshaler = &OrderedMap[int, any]{} +) + +// MarshalJSON implements the json.Marshaler interface. +func (om *OrderedMap[K, V]) MarshalJSON() ([]byte, error) { //nolint:funlen + if om == nil || om.list == nil { + return []byte("null"), nil + } + + var buf bytes.Buffer + buf.WriteByte('{') + enc := json.NewEncoder(&buf) + for pair, firstIteration := om.Oldest(), true; pair != nil; pair = pair.Next() { + if firstIteration { + firstIteration = false + } else { + buf.WriteByte(',') + } + + switch key := any(pair.Key).(type) { + case string, encoding.TextMarshaler: + if err := enc.Encode(pair.Key); err != nil { + return nil, err + } + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + buf.WriteByte('"') + buf.WriteString(fmt.Sprint(key)) + buf.WriteByte('"') + default: + // this switch takes care of wrapper types around primitive types, such as + // type myType string + switch keyValue := reflect.ValueOf(key); keyValue.Type().Kind() { + case reflect.String: + if err := enc.Encode(pair.Key); err != nil { + return nil, err + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + buf.WriteByte('"') + buf.WriteString(fmt.Sprint(key)) + buf.WriteByte('"') + default: + return nil, fmt.Errorf("unsupported key type: %T", key) + } + } + + buf.WriteByte(':') + if err := enc.Encode(pair.Value); err != nil { + return nil, err + } + } + buf.WriteByte('}') + return buf.Bytes(), nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (om *OrderedMap[K, V]) UnmarshalJSON(data []byte) error { + if om.list == nil { + om.initialize(0) + } + + d := json.NewDecoder(bytes.NewReader(data)) + tok, err := d.Token() + if err != nil { + return err + } + if tok != json.Delim('{') { + return errors.New("expect JSON object open with '{'") + } + + for d.More() { + // key + tok, err = d.Token() + if err != nil { + return err + } + + keyStr, ok := tok.(string) + if !ok { + return fmt.Errorf("key must be a string, got %T\n", tok) + } + + var key K + switch typedKey := any(&key).(type) { + case *string: + *typedKey = keyStr + case encoding.TextUnmarshaler: + err = typedKey.UnmarshalText([]byte(keyStr)) + case *int, *int8, *int16, *int32, *int64, *uint, *uint8, *uint16, *uint32, *uint64: + err = json.Unmarshal([]byte(keyStr), typedKey) + default: + // this switch takes care of wrapper types around primitive types, such as + // type myType string + switch reflect.TypeOf(key).Kind() { + case reflect.String: + convertedKeyData := reflect.ValueOf(keyStr).Convert(reflect.TypeOf(key)) + reflect.ValueOf(&key).Elem().Set(convertedKeyData) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + err = json.Unmarshal([]byte(keyStr), &key) + default: + err = fmt.Errorf("unsupported key type: %T", key) + } + } + if err != nil { + return err + } + + // value + value, _ := om.Get(key) + err = d.Decode(&value) + if err != nil { + return err + } + om.Set(key, value) + } + + tok, err = d.Token() + if err != nil { + return err + } + if tok != json.Delim('}') { + return errors.New("expect JSON object close with '}'") + } + return nil +} diff --git a/common/orderedmap/json_fuzz_test.go b/common/orderedmap/json_fuzz_test.go new file mode 100644 index 00000000..f556cea4 --- /dev/null +++ b/common/orderedmap/json_fuzz_test.go @@ -0,0 +1,117 @@ +package orderedmap + +// Adapted from https://github.com/dvyukov/go-fuzz-corpus/blob/c42c1b2/json/json.go + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func FuzzRoundTripJSON(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + for _, testCase := range []struct { + name string + constructor func() any + // should be a function that asserts that 2 objects of the type returned by constructor are equal + equalityAssertion func(*testing.T, any, any) bool + }{ + { + name: "with a string -> string map", + constructor: func() any { return &OrderedMap[string, string]{} }, + equalityAssertion: assertOrderedMapsEqual[string, string], + }, + { + name: "with a string -> int map", + constructor: func() any { return &OrderedMap[string, int]{} }, + equalityAssertion: assertOrderedMapsEqual[string, int], + }, + { + name: "with a string -> any map", + constructor: func() any { return &OrderedMap[string, any]{} }, + equalityAssertion: assertOrderedMapsEqual[string, any], + }, + { + name: "with a struct with map fields", + constructor: func() any { return new(testFuzzStruct) }, + equalityAssertion: assertTestFuzzStructEqual, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + v1 := testCase.constructor() + if json.Unmarshal(data, v1) != nil { + return + } + + jsonData, err := json.Marshal(v1) + require.NoError(t, err) + + v2 := testCase.constructor() + require.NoError(t, json.Unmarshal(jsonData, v2)) + + if !assert.True(t, testCase.equalityAssertion(t, v1, v2), "failed with input data %q", string(data)) { + // look at that what the standard lib does with regular map, to help with debugging + + var m1 map[string]any + require.NoError(t, json.Unmarshal(data, &m1)) + + mapJsonData, err := json.Marshal(m1) + require.NoError(t, err) + + var m2 map[string]any + require.NoError(t, json.Unmarshal(mapJsonData, &m2)) + + t.Logf("initial data = %s", string(data)) + t.Logf("unmarshalled map = %v", m1) + t.Logf("re-marshalled from map = %s", string(mapJsonData)) + t.Logf("re-marshalled from test obj = %s", string(jsonData)) + t.Logf("re-unmarshalled map = %s", m2) + } + }) + } + }) +} + +// only works for fairly basic maps, that's why it's just in this file +func assertOrderedMapsEqual[K comparable, V any](t *testing.T, v1, v2 any) bool { + om1, ok1 := v1.(*OrderedMap[K, V]) + om2, ok2 := v2.(*OrderedMap[K, V]) + + if !assert.True(t, ok1, "v1 not an orderedmap") || + !assert.True(t, ok2, "v2 not an orderedmap") { + return false + } + + success := assert.Equal(t, om1.Len(), om2.Len(), "om1 and om2 have different lengths: %d vs %d", om1.Len(), om2.Len()) + + for i, pair1, pair2 := 0, om1.Oldest(), om2.Oldest(); pair1 != nil && pair2 != nil; i, pair1, pair2 = i+1, pair1.Next(), pair2.Next() { + success = assert.Equal(t, pair1.Key, pair2.Key, "different keys at position %d: %v vs %v", i, pair1.Key, pair2.Key) && success + success = assert.Equal(t, pair1.Value, pair2.Value, "different values at position %d: %v vs %v", i, pair1.Value, pair2.Value) && success + } + + return success +} + +type testFuzzStruct struct { + M1 *OrderedMap[int, any] + M2 *OrderedMap[int, string] + M3 *OrderedMap[string, string] +} + +func assertTestFuzzStructEqual(t *testing.T, v1, v2 any) bool { + s1, ok := v1.(*testFuzzStruct) + s2, ok := v2.(*testFuzzStruct) + + if !assert.True(t, ok, "v1 not an testFuzzStruct") || + !assert.True(t, ok, "v2 not an testFuzzStruct") { + return false + } + + success := assertOrderedMapsEqual[int, any](t, s1.M1, s2.M1) + success = assertOrderedMapsEqual[int, string](t, s1.M2, s2.M2) && success + success = assertOrderedMapsEqual[string, string](t, s1.M3, s2.M3) && success + + return success +} diff --git a/common/orderedmap/json_test.go b/common/orderedmap/json_test.go new file mode 100644 index 00000000..42b89ab3 --- /dev/null +++ b/common/orderedmap/json_test.go @@ -0,0 +1,338 @@ +package orderedmap + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// to test marshalling TextMarshalers and unmarshalling TextUnmarshalers +type marshallable int + +func (m marshallable) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("#%d#", m)), nil +} + +func (m *marshallable) UnmarshalText(text []byte) error { + if len(text) < 3 { + return errors.New("too short") + } + if text[0] != '#' || text[len(text)-1] != '#' { + return errors.New("missing prefix or suffix") + } + + value, err := strconv.Atoi(string(text[1 : len(text)-1])) + if err != nil { + return err + } + + *m = marshallable(value) + return nil +} + +func TestMarshalJSON(t *testing.T) { + t.Run("int key", func(t *testing.T) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(8, "baz") + om.Set(8, "baz") + om.Set(9, "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque auctor augue accumsan mi maximus, quis viverra massa pretium. Phasellus imperdiet sapien a interdum sollicitudin. Duis at commodo lectus, a lacinia sem.") + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{"1":"bar","7":"baz","2":28,"3":100,"4":"baz","5":"28","6":"100","8":"baz","9":"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque auctor augue accumsan mi maximus, quis viverra massa pretium. Phasellus imperdiet sapien a interdum sollicitudin. Duis at commodo lectus, a lacinia sem."}`, string(b)) + }) + + t.Run("string key", func(t *testing.T) { + om := New[string, any]() + om.Set("test", "bar") + om.Set("abc", true) + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{"test":"bar","abc":true}`, string(b)) + }) + + t.Run("typed string key", func(t *testing.T) { + type myString string + om := New[myString, any]() + om.Set("test", "bar") + om.Set("abc", true) + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{"test":"bar","abc":true}`, string(b)) + }) + + t.Run("typed int key", func(t *testing.T) { + type myInt uint32 + om := New[myInt, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{"1":"bar","7":"baz","2":28,"3":100,"4":"baz"}`, string(b)) + }) + + t.Run("TextMarshaller key", func(t *testing.T) { + om := New[marshallable, any]() + om.Set(marshallable(1), "bar") + om.Set(marshallable(28), true) + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{"#1#":"bar","#28#":true}`, string(b)) + }) + + t.Run("empty map", func(t *testing.T) { + om := New[string, any]() + + b, err := json.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `{}`, string(b)) + }) +} + +func TestUnmarshallJSON(t *testing.T) { + t.Run("int key", func(t *testing.T) { + data := `{"1":"bar","7":"baz","2":28,"3":100,"4":"baz","5":"28","6":"100","8":"baz"}` + + om := New[int, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []int{1, 7, 2, 3, 4, 5, 6, 8}, + []any{"bar", "baz", float64(28), float64(100), "baz", "28", "100", "baz"}) + }) + + t.Run("string key", func(t *testing.T) { + data := `{"test":"bar","abc":true}` + + om := New[string, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []string{"test", "abc"}, + []any{"bar", true}) + }) + + t.Run("typed string key", func(t *testing.T) { + data := `{"test":"bar","abc":true}` + + type myString string + om := New[myString, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []myString{"test", "abc"}, + []any{"bar", true}) + }) + + t.Run("typed int key", func(t *testing.T) { + data := `{"1":"bar","7":"baz","2":28,"3":100,"4":"baz","5":"28","6":"100","8":"baz"}` + + type myInt uint32 + om := New[myInt, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []myInt{1, 7, 2, 3, 4, 5, 6, 8}, + []any{"bar", "baz", float64(28), float64(100), "baz", "28", "100", "baz"}) + }) + + t.Run("TextUnmarshaler key", func(t *testing.T) { + data := `{"#1#":"bar","#28#":true}` + + om := New[marshallable, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []marshallable{1, 28}, + []any{"bar", true}) + }) + + t.Run("when fed with an input that's not an object", func(t *testing.T) { + for _, data := range []string{"true", `["foo"]`, "42", `"foo"`} { + om := New[int, any]() + require.Error(t, json.Unmarshal([]byte(data), &om)) + } + }) + + t.Run("empty map", func(t *testing.T) { + data := `{}` + + om := New[int, any]() + require.NoError(t, json.Unmarshal([]byte(data), &om)) + + assertLenEqual(t, om, 0) + }) +} + +// const specialCharacters = "\\\\/\"\b\f\n\r\t\x00\uffff\ufffd世界\u007f\u00ff\U0010FFFF" +const specialCharacters = "\uffff\ufffd世界\u007f\u00ff\U0010FFFF" + +func TestJSONSpecialCharacters(t *testing.T) { + baselineMap := map[string]any{specialCharacters: specialCharacters} + baselineData, err := json.Marshal(baselineMap) + require.NoError(t, err) // baseline proves this key is supported by official json library + t.Logf("specialCharacters: %#v as []rune:%v", specialCharacters, []rune(specialCharacters)) + t.Logf("baseline json data: %s", baselineData) + + t.Run("marshal special characters", func(t *testing.T) { + om := New[string, any]() + om.Set(specialCharacters, specialCharacters) + b, err := json.Marshal(om) + require.NoError(t, err) + require.Equal(t, baselineData, b) + + type myString string + om2 := New[myString, myString]() + om2.Set(specialCharacters, specialCharacters) + b, err = json.Marshal(om2) + require.NoError(t, err) + require.Equal(t, baselineData, b) + }) + + t.Run("unmarshall special characters", func(t *testing.T) { + om := New[string, any]() + require.NoError(t, json.Unmarshal(baselineData, &om)) + assertOrderedPairsEqual(t, om, + []string{specialCharacters}, + []any{specialCharacters}) + + type myString string + om2 := New[myString, myString]() + require.NoError(t, json.Unmarshal(baselineData, &om2)) + assertOrderedPairsEqual(t, om2, + []myString{specialCharacters}, + []myString{specialCharacters}) + }) +} + +// to test structs that have nested map fields +type nestedMaps struct { + X int `json:"x" yaml:"x"` + M *OrderedMap[string, []*OrderedMap[int, *OrderedMap[string, any]]] `json:"m" yaml:"m"` +} + +func TestJSONRoundTrip(t *testing.T) { + for _, testCase := range []struct { + name string + input string + targetFactory func() any + isPrettyPrinted bool + }{ + { + name: "", + input: `{ + "x": 28, + "m": { + "foo": [ + { + "12": { + "i": 12, + "b": true, + "n": null, + "m": { + "a": "b", + "c": 28 + } + }, + "28": { + "a": false, + "b": [ + 1, + 2, + 3 + ] + } + }, + { + "3": { + "c": null, + "d": 87 + }, + "4": { + "e": true + }, + "5": { + "f": 4, + "g": 5, + "h": 6 + } + } + ], + "bar": [ + { + "5": { + "foo": "bar" + } + } + ] + } +}`, + targetFactory: func() any { return &nestedMaps{} }, + isPrettyPrinted: true, + }, + { + name: "with UTF-8 special chars in key", + input: `{"�":0}`, + targetFactory: func() any { return &OrderedMap[string, int]{} }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + target := testCase.targetFactory() + + require.NoError(t, json.Unmarshal([]byte(testCase.input), target)) + + var ( + out []byte + err error + ) + if testCase.isPrettyPrinted { + out, err = json.MarshalIndent(target, "", " ") + } else { + out, err = json.Marshal(target) + } + + if assert.NoError(t, err) { + assert.Equal(t, strings.TrimSpace(testCase.input), string(out)) + } + }) + } +} + +func BenchmarkMarshalJSON(b *testing.B) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(8, "baz") + om.Set(8, "baz") + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = json.Marshal(om) + } +} diff --git a/common/orderedmap/orderedmap.go b/common/orderedmap/orderedmap.go new file mode 100644 index 00000000..79da17be --- /dev/null +++ b/common/orderedmap/orderedmap.go @@ -0,0 +1,295 @@ +// Package orderedmap implements an ordered map, i.e. a map that also keeps track of +// the order in which keys were inserted. +// +// All operations are constant-time. +// +// Github repo: https://github.com/wk8/go-ordered-map +package orderedmap + +import ( + "fmt" + + list "github.com/bahlo/generic-list-go" +) + +type Pair[K comparable, V any] struct { + Key K + Value V + + element *list.Element[*Pair[K, V]] +} + +type OrderedMap[K comparable, V any] struct { + pairs map[K]*Pair[K, V] + list *list.List[*Pair[K, V]] +} + +type initConfig[K comparable, V any] struct { + capacity int + initialData []Pair[K, V] +} + +type InitOption[K comparable, V any] func(config *initConfig[K, V]) + +// WithCapacity allows giving a capacity hint for the map, akin to the standard make(map[K]V, capacity). +func WithCapacity[K comparable, V any](capacity int) InitOption[K, V] { + return func(c *initConfig[K, V]) { + c.capacity = capacity + } +} + +// WithInitialData allows passing in initial data for the map. +func WithInitialData[K comparable, V any](initialData ...Pair[K, V]) InitOption[K, V] { + return func(c *initConfig[K, V]) { + c.initialData = initialData + if c.capacity < len(initialData) { + c.capacity = len(initialData) + } + } +} + +// New creates a new OrderedMap. +// options can either be one or several InitOption[K, V], or a single integer, +// which is then interpreted as a capacity hint, à la make(map[K]V, capacity). +func New[K comparable, V any](options ...any) *OrderedMap[K, V] { //nolint:varnamelen + orderedMap := &OrderedMap[K, V]{} + + var config initConfig[K, V] + for _, untypedOption := range options { + switch option := untypedOption.(type) { + case int: + if len(options) != 1 { + invalidOption() + } + config.capacity = option + + case InitOption[K, V]: + option(&config) + + default: + invalidOption() + } + } + + orderedMap.initialize(config.capacity) + orderedMap.AddPairs(config.initialData...) + + return orderedMap +} + +const invalidOptionMessage = `when using orderedmap.New[K,V]() with options, either provide one or several InitOption[K, V]; or a single integer which is then interpreted as a capacity hint, à la make(map[K]V, capacity).` //nolint:lll + +func invalidOption() { panic(invalidOptionMessage) } + +func (om *OrderedMap[K, V]) initialize(capacity int) { + om.pairs = make(map[K]*Pair[K, V], capacity) + om.list = list.New[*Pair[K, V]]() +} + +// Get looks for the given key, and returns the value associated with it, +// or V's nil value if not found. The boolean it returns says whether the key is present in the map. +func (om *OrderedMap[K, V]) Get(key K) (val V, present bool) { + if pair, present := om.pairs[key]; present { + return pair.Value, true + } + + return +} + +// Load is an alias for Get, mostly to present an API similar to `sync.Map`'s. +func (om *OrderedMap[K, V]) Load(key K) (V, bool) { + return om.Get(key) +} + +// Value returns the value associated with the given key or the zero value. +func (om *OrderedMap[K, V]) Value(key K) (val V) { + if pair, present := om.pairs[key]; present { + val = pair.Value + } + return +} + +// GetPair looks for the given key, and returns the pair associated with it, +// or nil if not found. The Pair struct can then be used to iterate over the ordered map +// from that point, either forward or backward. +func (om *OrderedMap[K, V]) GetPair(key K) *Pair[K, V] { + return om.pairs[key] +} + +// Set sets the key-value pair, and returns what `Get` would have returned +// on that key prior to the call to `Set`. +func (om *OrderedMap[K, V]) Set(key K, value V) (val V, present bool) { + if pair, present := om.pairs[key]; present { + oldValue := pair.Value + pair.Value = value + return oldValue, true + } + + pair := &Pair[K, V]{ + Key: key, + Value: value, + } + pair.element = om.list.PushBack(pair) + om.pairs[key] = pair + + return +} + +// AddPairs allows setting multiple pairs at a time. It's equivalent to calling +// Set on each pair sequentially. +func (om *OrderedMap[K, V]) AddPairs(pairs ...Pair[K, V]) { + for _, pair := range pairs { + om.Set(pair.Key, pair.Value) + } +} + +// Store is an alias for Set, mostly to present an API similar to `sync.Map`'s. +func (om *OrderedMap[K, V]) Store(key K, value V) (V, bool) { + return om.Set(key, value) +} + +// Delete removes the key-value pair, and returns what `Get` would have returned +// on that key prior to the call to `Delete`. +func (om *OrderedMap[K, V]) Delete(key K) (val V, present bool) { + if pair, present := om.pairs[key]; present { + om.list.Remove(pair.element) + delete(om.pairs, key) + return pair.Value, true + } + return +} + +// Len returns the length of the ordered map. +func (om *OrderedMap[K, V]) Len() int { + if om == nil || om.pairs == nil { + return 0 + } + return len(om.pairs) +} + +// Oldest returns a pointer to the oldest pair. It's meant to be used to iterate on the ordered map's +// pairs from the oldest to the newest, e.g.: +// for pair := orderedMap.Oldest(); pair != nil; pair = pair.Next() { fmt.Printf("%v => %v\n", pair.Key, pair.Value) } +func (om *OrderedMap[K, V]) Oldest() *Pair[K, V] { + if om == nil || om.list == nil { + return nil + } + return listElementToPair(om.list.Front()) +} + +// Newest returns a pointer to the newest pair. It's meant to be used to iterate on the ordered map's +// pairs from the newest to the oldest, e.g.: +// for pair := orderedMap.Oldest(); pair != nil; pair = pair.Next() { fmt.Printf("%v => %v\n", pair.Key, pair.Value) } +func (om *OrderedMap[K, V]) Newest() *Pair[K, V] { + if om == nil || om.list == nil { + return nil + } + return listElementToPair(om.list.Back()) +} + +// Next returns a pointer to the next pair. +func (p *Pair[K, V]) Next() *Pair[K, V] { + return listElementToPair(p.element.Next()) +} + +// Prev returns a pointer to the previous pair. +func (p *Pair[K, V]) Prev() *Pair[K, V] { + return listElementToPair(p.element.Prev()) +} + +func listElementToPair[K comparable, V any](element *list.Element[*Pair[K, V]]) *Pair[K, V] { + if element == nil { + return nil + } + return element.Value +} + +// KeyNotFoundError may be returned by functions in this package when they're called with keys that are not present +// in the map. +type KeyNotFoundError[K comparable] struct { + MissingKey K +} + +func (e *KeyNotFoundError[K]) Error() string { + return fmt.Sprintf("missing key: %v", e.MissingKey) +} + +// MoveAfter moves the value associated with key to its new position after the one associated with markKey. +// Returns an error iff key or markKey are not present in the map. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) MoveAfter(key, markKey K) error { + elements, err := om.getElements(key, markKey) + if err != nil { + return err + } + om.list.MoveAfter(elements[0], elements[1]) + return nil +} + +// MoveBefore moves the value associated with key to its new position before the one associated with markKey. +// Returns an error iff key or markKey are not present in the map. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) MoveBefore(key, markKey K) error { + elements, err := om.getElements(key, markKey) + if err != nil { + return err + } + om.list.MoveBefore(elements[0], elements[1]) + return nil +} + +func (om *OrderedMap[K, V]) getElements(keys ...K) ([]*list.Element[*Pair[K, V]], error) { + elements := make([]*list.Element[*Pair[K, V]], len(keys)) + for i, k := range keys { + pair, present := om.pairs[k] + if !present { + return nil, &KeyNotFoundError[K]{k} + } + elements[i] = pair.element + } + return elements, nil +} + +// MoveToBack moves the value associated with key to the back of the ordered map, +// i.e. makes it the newest pair in the map. +// Returns an error iff key is not present in the map. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) MoveToBack(key K) error { + _, err := om.GetAndMoveToBack(key) + return err +} + +// MoveToFront moves the value associated with key to the front of the ordered map, +// i.e. makes it the oldest pair in the map. +// Returns an error iff key is not present in the map. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) MoveToFront(key K) error { + _, err := om.GetAndMoveToFront(key) + return err +} + +// GetAndMoveToBack combines Get and MoveToBack in the same call. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) GetAndMoveToBack(key K) (val V, err error) { + if pair, present := om.pairs[key]; present { + val = pair.Value + om.list.MoveToBack(pair.element) + } else { + err = &KeyNotFoundError[K]{key} + } + + return +} + +// GetAndMoveToFront combines Get and MoveToFront in the same call. If an error is returned, +// it will be a KeyNotFoundError. +func (om *OrderedMap[K, V]) GetAndMoveToFront(key K) (val V, err error) { + if pair, present := om.pairs[key]; present { + val = pair.Value + om.list.MoveToFront(pair.element) + } else { + err = &KeyNotFoundError[K]{key} + } + + return +} diff --git a/common/orderedmap/orderedmap_test.go b/common/orderedmap/orderedmap_test.go new file mode 100644 index 00000000..a3137eb0 --- /dev/null +++ b/common/orderedmap/orderedmap_test.go @@ -0,0 +1,384 @@ +package orderedmap + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicFeatures(t *testing.T) { + n := 100 + om := New[int, int]() + + // set(i, 2 * i) + for i := 0; i < n; i++ { + assertLenEqual(t, om, i) + oldValue, present := om.Set(i, 2*i) + assertLenEqual(t, om, i+1) + + assert.Equal(t, 0, oldValue) + assert.False(t, present) + } + + // get what we just set + for i := 0; i < n; i++ { + value, present := om.Get(i) + + assert.Equal(t, 2*i, value) + assert.Equal(t, value, om.Value(i)) + assert.True(t, present) + } + + // get pairs of what we just set + for i := 0; i < n; i++ { + pair := om.GetPair(i) + + assert.NotNil(t, pair) + assert.Equal(t, 2*i, pair.Value) + } + + // forward iteration + i := 0 + for pair := om.Oldest(); pair != nil; pair = pair.Next() { + assert.Equal(t, i, pair.Key) + assert.Equal(t, 2*i, pair.Value) + i++ + } + // backward iteration + i = n - 1 + for pair := om.Newest(); pair != nil; pair = pair.Prev() { + assert.Equal(t, i, pair.Key) + assert.Equal(t, 2*i, pair.Value) + i-- + } + + // forward iteration starting from known key + i = 42 + for pair := om.GetPair(i); pair != nil; pair = pair.Next() { + assert.Equal(t, i, pair.Key) + assert.Equal(t, 2*i, pair.Value) + i++ + } + + // double values for pairs with even keys + for j := 0; j < n/2; j++ { + i = 2 * j + oldValue, present := om.Set(i, 4*i) + + assert.Equal(t, 2*i, oldValue) + assert.True(t, present) + } + // and delete pairs with odd keys + for j := 0; j < n/2; j++ { + i = 2*j + 1 + assertLenEqual(t, om, n-j) + value, present := om.Delete(i) + assertLenEqual(t, om, n-j-1) + + assert.Equal(t, 2*i, value) + assert.True(t, present) + + // deleting again shouldn't change anything + value, present = om.Delete(i) + assertLenEqual(t, om, n-j-1) + assert.Equal(t, 0, value) + assert.False(t, present) + } + + // get the whole range + for j := 0; j < n/2; j++ { + i = 2 * j + value, present := om.Get(i) + assert.Equal(t, 4*i, value) + assert.Equal(t, value, om.Value(i)) + assert.True(t, present) + + i = 2*j + 1 + value, present = om.Get(i) + assert.Equal(t, 0, value) + assert.Equal(t, value, om.Value(i)) + assert.False(t, present) + } + + // check iterations again + i = 0 + for pair := om.Oldest(); pair != nil; pair = pair.Next() { + assert.Equal(t, i, pair.Key) + assert.Equal(t, 4*i, pair.Value) + i += 2 + } + i = 2 * ((n - 1) / 2) + for pair := om.Newest(); pair != nil; pair = pair.Prev() { + assert.Equal(t, i, pair.Key) + assert.Equal(t, 4*i, pair.Value) + i -= 2 + } +} + +func TestUpdatingDoesntChangePairsOrder(t *testing.T) { + om := New[string, any]() + om.Set("foo", "bar") + om.Set("wk", 28) + om.Set("po", 100) + om.Set("bar", "baz") + + oldValue, present := om.Set("po", 102) + assert.Equal(t, 100, oldValue) + assert.True(t, present) + + assertOrderedPairsEqual(t, om, + []string{"foo", "wk", "po", "bar"}, + []any{"bar", 28, 102, "baz"}) +} + +func TestDeletingAndReinsertingChangesPairsOrder(t *testing.T) { + om := New[string, any]() + om.Set("foo", "bar") + om.Set("wk", 28) + om.Set("po", 100) + om.Set("bar", "baz") + + // delete a pair + oldValue, present := om.Delete("po") + assert.Equal(t, 100, oldValue) + assert.True(t, present) + + // re-insert the same pair + oldValue, present = om.Set("po", 100) + assert.Nil(t, oldValue) + assert.False(t, present) + + assertOrderedPairsEqual(t, om, + []string{"foo", "wk", "bar", "po"}, + []any{"bar", 28, "baz", 100}) +} + +func TestEmptyMapOperations(t *testing.T) { + om := New[string, any]() + + oldValue, present := om.Get("foo") + assert.Nil(t, oldValue) + assert.Nil(t, om.Value("foo")) + assert.False(t, present) + + oldValue, present = om.Delete("bar") + assert.Nil(t, oldValue) + assert.False(t, present) + + assertLenEqual(t, om, 0) + + assert.Nil(t, om.Oldest()) + assert.Nil(t, om.Newest()) +} + +type dummyTestStruct struct { + value string +} + +func TestPackUnpackStructs(t *testing.T) { + om := New[string, dummyTestStruct]() + om.Set("foo", dummyTestStruct{"foo!"}) + om.Set("bar", dummyTestStruct{"bar!"}) + + value, present := om.Get("foo") + assert.True(t, present) + assert.Equal(t, value, om.Value("foo")) + if assert.NotNil(t, value) { + assert.Equal(t, "foo!", value.value) + } + + value, present = om.Set("bar", dummyTestStruct{"baz!"}) + assert.True(t, present) + if assert.NotNil(t, value) { + assert.Equal(t, "bar!", value.value) + } + + value, present = om.Get("bar") + assert.Equal(t, value, om.Value("bar")) + assert.True(t, present) + if assert.NotNil(t, value) { + assert.Equal(t, "baz!", value.value) + } +} + +// shamelessly stolen from https://github.com/python/cpython/blob/e19a91e45fd54a56e39c2d12e6aaf4757030507f/Lib/test/test_ordered_dict.py#L55-L61 +func TestShuffle(t *testing.T) { + ranLen := 100 + + for _, n := range []int{0, 10, 20, 100, 1000, 10000} { + t.Run(fmt.Sprintf("shuffle test with %d items", n), func(t *testing.T) { + om := New[string, string]() + + keys := make([]string, n) + values := make([]string, n) + + for i := 0; i < n; i++ { + // we prefix with the number to ensure that we don't get any duplicates + keys[i] = fmt.Sprintf("%d_%s", i, randomHexString(t, ranLen)) + values[i] = randomHexString(t, ranLen) + + value, present := om.Set(keys[i], values[i]) + assert.Equal(t, "", value) + assert.False(t, present) + } + + assertOrderedPairsEqual(t, om, keys, values) + }) + } +} + +func TestMove(t *testing.T) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(7, "baz") + om.Set(8, "baz") + + err := om.MoveAfter(2, 3) + assert.Nil(t, err) + assertOrderedPairsEqual(t, om, + []int{1, 3, 2, 4, 5, 6, 7, 8}, + []any{"bar", 100, 28, "baz", "28", "100", "baz", "baz"}) + + err = om.MoveBefore(6, 4) + assert.Nil(t, err) + assertOrderedPairsEqual(t, om, + []int{1, 3, 2, 6, 4, 5, 7, 8}, + []any{"bar", 100, 28, "100", "baz", "28", "baz", "baz"}) + + err = om.MoveToBack(3) + assert.Nil(t, err) + assertOrderedPairsEqual(t, om, + []int{1, 2, 6, 4, 5, 7, 8, 3}, + []any{"bar", 28, "100", "baz", "28", "baz", "baz", 100}) + + err = om.MoveToFront(5) + assert.Nil(t, err) + assertOrderedPairsEqual(t, om, + []int{5, 1, 2, 6, 4, 7, 8, 3}, + []any{"28", "bar", 28, "100", "baz", "baz", "baz", 100}) + + err = om.MoveToFront(100) + assert.Equal(t, &KeyNotFoundError[int]{100}, err) +} + +func TestGetAndMove(t *testing.T) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(7, "baz") + om.Set(8, "baz") + + value, err := om.GetAndMoveToBack(3) + assert.Nil(t, err) + assert.Equal(t, 100, value) + assertOrderedPairsEqual(t, om, + []int{1, 2, 4, 5, 6, 7, 8, 3}, + []any{"bar", 28, "baz", "28", "100", "baz", "baz", 100}) + + value, err = om.GetAndMoveToFront(5) + assert.Nil(t, err) + assert.Equal(t, "28", value) + assertOrderedPairsEqual(t, om, + []int{5, 1, 2, 4, 6, 7, 8, 3}, + []any{"28", "bar", 28, "baz", "100", "baz", "baz", 100}) + + value, err = om.GetAndMoveToBack(100) + assert.Equal(t, &KeyNotFoundError[int]{100}, err) +} + +func TestAddPairs(t *testing.T) { + om := New[int, any]() + om.AddPairs( + Pair[int, any]{ + Key: 28, + Value: "foo", + }, + Pair[int, any]{ + Key: 12, + Value: "bar", + }, + Pair[int, any]{ + Key: 28, + Value: "baz", + }, + ) + + assertOrderedPairsEqual(t, om, + []int{28, 12}, + []any{"baz", "bar"}) +} + +// sadly, we can't test the "actual" capacity here, see https://github.com/golang/go/issues/52157 +func TestNewWithCapacity(t *testing.T) { + zero := New[int, string](0) + assert.Empty(t, zero.Len()) + + assert.PanicsWithValue(t, invalidOptionMessage, func() { + _ = New[int, string](1, 2) + }) + assert.PanicsWithValue(t, invalidOptionMessage, func() { + _ = New[int, string](1, 2, 3) + }) + + om := New[int, string](-1) + om.Set(1337, "quarante-deux") + assert.Equal(t, 1, om.Len()) +} + +func TestNewWithOptions(t *testing.T) { + t.Run("wih capacity", func(t *testing.T) { + om := New[string, any](WithCapacity[string, any](98)) + assert.Equal(t, 0, om.Len()) + }) + + t.Run("with initial data", func(t *testing.T) { + om := New[string, int](WithInitialData( + Pair[string, int]{ + Key: "a", + Value: 1, + }, + Pair[string, int]{ + Key: "b", + Value: 2, + }, + Pair[string, int]{ + Key: "c", + Value: 3, + }, + )) + + assertOrderedPairsEqual(t, om, + []string{"a", "b", "c"}, + []int{1, 2, 3}) + }) + + t.Run("with an invalid option type", func(t *testing.T) { + assert.PanicsWithValue(t, invalidOptionMessage, func() { + _ = New[int, string]("foo") + }) + }) +} + +func TestNilMap(t *testing.T) { + // we want certain behaviors of a nil ordered map to be the same as they are for standard nil maps + var om *OrderedMap[int, any] + + t.Run("len", func(t *testing.T) { + assert.Equal(t, 0, om.Len()) + }) + + t.Run("iterating - akin to range", func(t *testing.T) { + assert.Nil(t, om.Oldest()) + assert.Nil(t, om.Newest()) + }) +} diff --git a/common/orderedmap/utils_test.go b/common/orderedmap/utils_test.go new file mode 100644 index 00000000..9f1cc3bf --- /dev/null +++ b/common/orderedmap/utils_test.go @@ -0,0 +1,76 @@ +package orderedmap + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// assertOrderedPairsEqual asserts that the map contains the given keys and values +// from oldest to newest. +func assertOrderedPairsEqual[K comparable, V any]( + t *testing.T, orderedMap *OrderedMap[K, V], expectedKeys []K, expectedValues []V, +) { + t.Helper() + + assertOrderedPairsEqualFromNewest(t, orderedMap, expectedKeys, expectedValues) + assertOrderedPairsEqualFromOldest(t, orderedMap, expectedKeys, expectedValues) +} + +func assertOrderedPairsEqualFromNewest[K comparable, V any]( + t *testing.T, orderedMap *OrderedMap[K, V], expectedKeys []K, expectedValues []V, +) { + t.Helper() + + if assert.Equal(t, len(expectedKeys), len(expectedValues)) && assert.Equal(t, len(expectedKeys), orderedMap.Len()) { + i := orderedMap.Len() - 1 + for pair := orderedMap.Newest(); pair != nil; pair = pair.Prev() { + assert.Equal(t, expectedKeys[i], pair.Key, "from newest index=%d on key", i) + assert.Equal(t, expectedValues[i], pair.Value, "from newest index=%d on value", i) + i-- + } + } +} + +func assertOrderedPairsEqualFromOldest[K comparable, V any]( + t *testing.T, orderedMap *OrderedMap[K, V], expectedKeys []K, expectedValues []V, +) { + t.Helper() + + if assert.Equal(t, len(expectedKeys), len(expectedValues)) && assert.Equal(t, len(expectedKeys), orderedMap.Len()) { + i := 0 + for pair := orderedMap.Oldest(); pair != nil; pair = pair.Next() { + assert.Equal(t, expectedKeys[i], pair.Key, "from oldest index=%d on key", i) + assert.Equal(t, expectedValues[i], pair.Value, "from oldest index=%d on value", i) + i++ + } + } +} + +func assertLenEqual[K comparable, V any](t *testing.T, orderedMap *OrderedMap[K, V], expectedLen int) { + t.Helper() + + assert.Equal(t, expectedLen, orderedMap.Len()) + + // also check the list length, for good measure + assert.Equal(t, expectedLen, orderedMap.list.Len()) +} + +func randomHexString(t *testing.T, length int) string { + t.Helper() + + b := length / 2 //nolint:gomnd + randBytes := make([]byte, b) + + if n, err := rand.Read(randBytes); err != nil || n != b { + if err == nil { + err = fmt.Errorf("only got %v random bytes, expected %v", n, b) + } + t.Fatal(err) + } + + return hex.EncodeToString(randBytes) +} diff --git a/common/orderedmap/yaml.go b/common/orderedmap/yaml.go new file mode 100644 index 00000000..60224712 --- /dev/null +++ b/common/orderedmap/yaml.go @@ -0,0 +1,71 @@ +package orderedmap + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +var ( + _ yaml.Marshaler = &OrderedMap[int, any]{} + _ yaml.Unmarshaler = &OrderedMap[int, any]{} +) + +// MarshalYAML implements the yaml.Marshaler interface. +func (om *OrderedMap[K, V]) MarshalYAML() (interface{}, error) { + if om == nil { + return []byte("null"), nil + } + + node := yaml.Node{ + Kind: yaml.MappingNode, + } + + for pair := om.Oldest(); pair != nil; pair = pair.Next() { + key, value := pair.Key, pair.Value + + keyNode := &yaml.Node{} + + // serialize key to yaml, then deserialize it back into the node + // this is a hack to get the correct tag for the key + if err := keyNode.Encode(key); err != nil { + return nil, err + } + + valueNode := &yaml.Node{} + if err := valueNode.Encode(value); err != nil { + return nil, err + } + + node.Content = append(node.Content, keyNode, valueNode) + } + + return &node, nil +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (om *OrderedMap[K, V]) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.MappingNode { + return fmt.Errorf("pipeline must contain YAML mapping, has %v", value.Kind) + } + + if om.list == nil { + om.initialize(0) + } + + for index := 0; index < len(value.Content); index += 2 { + var key K + var val V + + if err := value.Content[index].Decode(&key); err != nil { + return err + } + if err := value.Content[index+1].Decode(&val); err != nil { + return err + } + + om.Set(key, val) + } + + return nil +} diff --git a/common/orderedmap/yaml_fuzz_test.go b/common/orderedmap/yaml_fuzz_test.go new file mode 100644 index 00000000..14e67549 --- /dev/null +++ b/common/orderedmap/yaml_fuzz_test.go @@ -0,0 +1,82 @@ +package orderedmap + +// Adapted from https://github.com/dvyukov/go-fuzz-corpus/blob/c42c1b2/json/json.go + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func FuzzRoundTripYAML(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + for _, testCase := range []struct { + name string + constructor func() any + // should be a function that asserts that 2 objects of the type returned by constructor are equal + equalityAssertion func(*testing.T, any, any) bool + }{ + { + name: "with a string -> string map", + constructor: func() any { return &OrderedMap[string, string]{} }, + equalityAssertion: assertOrderedMapsEqual[string, string], + }, + { + name: "with a string -> int map", + constructor: func() any { return &OrderedMap[string, int]{} }, + equalityAssertion: assertOrderedMapsEqual[string, int], + }, + { + name: "with a string -> any map", + constructor: func() any { return &OrderedMap[string, any]{} }, + equalityAssertion: assertOrderedMapsEqual[string, any], + }, + { + name: "with a struct with map fields", + constructor: func() any { return new(testFuzzStruct) }, + equalityAssertion: assertTestFuzzStructEqual, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + v1 := testCase.constructor() + if yaml.Unmarshal(data, v1) != nil { + return + } + t.Log(data) + t.Log(v1) + + yamlData, err := yaml.Marshal(v1) + require.NoError(t, err) + t.Log(string(yamlData)) + + v2 := testCase.constructor() + err = yaml.Unmarshal(yamlData, v2) + if err != nil { + t.Log(string(yamlData)) + t.Fatal(err) + } + + if !assert.True(t, testCase.equalityAssertion(t, v1, v2), "failed with input data %q", string(data)) { + // look at that what the standard lib does with regular map, to help with debugging + + var m1 map[string]any + require.NoError(t, yaml.Unmarshal(data, &m1)) + + mapJsonData, err := yaml.Marshal(m1) + require.NoError(t, err) + + var m2 map[string]any + require.NoError(t, yaml.Unmarshal(mapJsonData, &m2)) + + t.Logf("initial data = %s", string(data)) + t.Logf("unmarshalled map = %v", m1) + t.Logf("re-marshalled from map = %s", string(mapJsonData)) + t.Logf("re-marshalled from test obj = %s", string(yamlData)) + t.Logf("re-unmarshalled map = %s", m2) + } + }) + } + }) +} diff --git a/common/orderedmap/yaml_test.go b/common/orderedmap/yaml_test.go new file mode 100644 index 00000000..0ffbf64b --- /dev/null +++ b/common/orderedmap/yaml_test.go @@ -0,0 +1,334 @@ +package orderedmap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestMarshalYAML(t *testing.T) { + t.Run("int key", func(t *testing.T) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(8, "baz") + om.Set(8, "baz") + om.Set(9, "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque auctor augue accumsan mi maximus, quis viverra massa pretium. Phasellus imperdiet sapien a interdum sollicitudin. Duis at commodo lectus, a lacinia sem.") + + b, err := yaml.Marshal(om) + + expected := `1: bar +7: baz +2: 28 +3: 100 +4: baz +5: "28" +6: "100" +8: baz +9: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque auctor augue accumsan mi maximus, quis viverra massa pretium. Phasellus imperdiet sapien a interdum sollicitudin. Duis at commodo lectus, a lacinia sem. +` + assert.NoError(t, err) + assert.Equal(t, expected, string(b)) + }) + + t.Run("string key", func(t *testing.T) { + om := New[string, any]() + om.Set("test", "bar") + om.Set("abc", true) + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + expected := `test: bar +abc: true +` + assert.Equal(t, expected, string(b)) + }) + + t.Run("typed string key", func(t *testing.T) { + type myString string + om := New[myString, any]() + om.Set("test", "bar") + om.Set("abc", true) + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `test: bar +abc: true +`, string(b)) + }) + + t.Run("typed int key", func(t *testing.T) { + type myInt uint32 + om := New[myInt, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `1: bar +7: baz +2: 28 +3: 100 +4: baz +`, string(b)) + }) + + t.Run("TextMarshaller key", func(t *testing.T) { + om := New[marshallable, any]() + om.Set(marshallable(1), "bar") + om.Set(marshallable(28), true) + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, `'#1#': bar +'#28#': true +`, string(b)) + }) + + t.Run("empty map with 0 elements", func(t *testing.T) { + om := New[string, any]() + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, "{}\n", string(b)) + }) + + t.Run("empty map with no elements (null)", func(t *testing.T) { + om := &OrderedMap[string, string]{} + + b, err := yaml.Marshal(om) + assert.NoError(t, err) + assert.Equal(t, "{}\n", string(b)) + }) +} + +func TestUnmarshallYAML(t *testing.T) { + t.Run("int key", func(t *testing.T) { + data := ` +1: bar +7: baz +2: 28 +3: 100 +4: baz +5: "28" +6: "100" +8: baz +` + om := New[int, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []int{1, 7, 2, 3, 4, 5, 6, 8}, + []any{"bar", "baz", 28, 100, "baz", "28", "100", "baz"}) + + // serialize back to yaml to make sure things are equal + }) + + t.Run("string key", func(t *testing.T) { + data := `{"test":"bar","abc":true}` + + om := New[string, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []string{"test", "abc"}, + []any{"bar", true}) + }) + + t.Run("typed string key", func(t *testing.T) { + data := `{"test":"bar","abc":true}` + + type myString string + om := New[myString, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []myString{"test", "abc"}, + []any{"bar", true}) + }) + + t.Run("typed int key", func(t *testing.T) { + data := ` +1: bar +7: baz +2: 28 +3: 100 +4: baz +5: "28" +6: "100" +8: baz +` + type myInt uint32 + om := New[myInt, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []myInt{1, 7, 2, 3, 4, 5, 6, 8}, + []any{"bar", "baz", 28, 100, "baz", "28", "100", "baz"}) + }) + + t.Run("TextUnmarshaler key", func(t *testing.T) { + data := `{"#1#":"bar","#28#":true}` + + om := New[marshallable, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertOrderedPairsEqual(t, om, + []marshallable{1, 28}, + []any{"bar", true}) + }) + + t.Run("when fed with an input that's not an object", func(t *testing.T) { + for _, data := range []string{"true", `["foo"]`, "42", `"foo"`} { + om := New[int, any]() + require.Error(t, yaml.Unmarshal([]byte(data), &om)) + } + }) + + t.Run("empty map", func(t *testing.T) { + data := `{}` + + om := New[int, any]() + require.NoError(t, yaml.Unmarshal([]byte(data), &om)) + + assertLenEqual(t, om, 0) + }) +} + +func TestYAMLSpecialCharacters(t *testing.T) { + baselineMap := map[string]any{specialCharacters: specialCharacters} + baselineData, err := yaml.Marshal(baselineMap) + require.NoError(t, err) // baseline proves this key is supported by official yaml library + t.Logf("specialCharacters: %#v as []rune:%v", specialCharacters, []rune(specialCharacters)) + t.Logf("baseline yaml data: %s", baselineData) + + t.Run("marshal special characters", func(t *testing.T) { + om := New[string, any]() + om.Set(specialCharacters, specialCharacters) + b, err := yaml.Marshal(om) + require.NoError(t, err) + require.Equal(t, baselineData, b) + + type myString string + om2 := New[myString, myString]() + om2.Set(specialCharacters, specialCharacters) + b, err = yaml.Marshal(om2) + require.NoError(t, err) + require.Equal(t, baselineData, b) + }) + + t.Run("unmarshall special characters", func(t *testing.T) { + om := New[string, any]() + require.NoError(t, yaml.Unmarshal(baselineData, &om)) + assertOrderedPairsEqual(t, om, + []string{specialCharacters}, + []any{specialCharacters}) + + type myString string + om2 := New[myString, myString]() + require.NoError(t, yaml.Unmarshal(baselineData, &om2)) + assertOrderedPairsEqual(t, om2, + []myString{specialCharacters}, + []myString{specialCharacters}) + }) +} + +func TestYAMLRoundTrip(t *testing.T) { + for _, testCase := range []struct { + name string + input string + targetFactory func() any + }{ + { + name: "empty map", + input: "{}\n", + targetFactory: func() any { + return &OrderedMap[string, any]{} + }, + }, + { + name: "", + input: `x: 28 +m: + bar: + - 5: + foo: bar + foo: + - 12: + b: true + i: 12 + m: + a: b + c: 28 + "n": null + 28: + a: false + b: + - 1 + - 2 + - 3 + - 3: + c: null + d: 87 + 4: + e: true + 5: + f: 4 + g: 5 + h: 6 +`, + targetFactory: func() any { return &nestedMaps{} }, + }, + { + name: "with UTF-8 special chars in key", + input: "�: 0\n", + targetFactory: func() any { return &OrderedMap[string, int]{} }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + target := testCase.targetFactory() + + require.NoError(t, yaml.Unmarshal([]byte(testCase.input), target)) + + var ( + out []byte + err error + ) + + out, err = yaml.Marshal(target) + + if assert.NoError(t, err) { + assert.Equal(t, testCase.input, string(out)) + } + }) + } +} + +func BenchmarkMarshalYAML(b *testing.B) { + om := New[int, any]() + om.Set(1, "bar") + om.Set(7, "baz") + om.Set(2, 28) + om.Set(3, 100) + om.Set(4, "baz") + om.Set(5, "28") + om.Set(6, "100") + om.Set(8, "baz") + om.Set(8, "baz") + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = yaml.Marshal(om) + } +} diff --git a/config/config.go b/config/config.go index ab2dbdfc..67bc90bb 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,7 @@ import ( "github.com/metacubex/mihomo/adapter/outbound" "github.com/metacubex/mihomo/adapter/outboundgroup" "github.com/metacubex/mihomo/adapter/provider" + "github.com/metacubex/mihomo/common/orderedmap" "github.com/metacubex/mihomo/common/utils" "github.com/metacubex/mihomo/common/yaml" "github.com/metacubex/mihomo/component/auth" @@ -38,7 +39,6 @@ import ( RW "github.com/metacubex/mihomo/rules/wrapper" T "github.com/metacubex/mihomo/tunnel" - orderedmap "github.com/wk8/go-ordered-map/v2" "golang.org/x/exp/slices" ) diff --git a/go.mod b/go.mod index 9e696ced..360c2ec1 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,6 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 github.com/vmihailenco/msgpack/v5 v5.4.1 - github.com/wk8/go-ordered-map/v2 v2.1.8 gitlab.com/go-extension/aes-ccm v0.0.0-20230221065045-e58665ef23c7 go.uber.org/automaxprocs v1.6.0 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba @@ -70,7 +69,6 @@ require ( github.com/Yawning/aez v0.0.0-20211027044916-e49e68abd344 // indirect github.com/ajg/form v1.5.1 // indirect github.com/andybalholm/brotli v1.0.6 // indirect - github.com/buger/jsonparser v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ericlagergren/aegis v0.0.0-20250325060835-cd0defd64358 // indirect github.com/ericlagergren/polyval v0.0.0-20220411101811-e25bc10ba391 // indirect @@ -88,7 +86,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.12.3 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/metacubex/ascon v0.1.0 // indirect github.com/metacubex/gvisor v0.0.0-20251227095601-261ec1326fe8 // indirect diff --git a/go.sum b/go.sum index 6409cad3..90aa90f2 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,6 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc= github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -59,7 +57,6 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73 github.com/google/tink/go v1.6.1 h1:t7JHqO8Ath2w2ig5vjwQYJzhGEZymedQc90lQXUBa4I= github.com/insomniacslk/dhcp v0.0.0-20250109001534-8abf58130905 h1:q3OEI9RaN/wwcx+qgGo6ZaoJkCiDYe/gjDLfq7lQQF4= github.com/insomniacslk/dhcp v0.0.0-20250109001534-8abf58130905/go.mod h1:VvGYjkZoJyKqlmT1yzakUs4mfKMNB0XdODP0+rdml6k= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= @@ -72,8 +69,6 @@ github.com/klauspost/reedsolomon v1.12.3/go.mod h1:3K5rXwABAvzGeR01r6pWZieUALXO/ github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= @@ -198,8 +193,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= -github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+AIT3M4mfUVinOCPgf2uUWYFUzN0sM= gitlab.com/go-extension/aes-ccm v0.0.0-20230221065045-e58665ef23c7 h1:UNrDfkQqiEYzdMlNsVvBYOAJWZjdktqFE9tQh5BT2+4= gitlab.com/go-extension/aes-ccm v0.0.0-20230221065045-e58665ef23c7/go.mod h1:E+rxHvJG9H6PUdzq9NRG6csuLN3XUx98BfGOVWNYnXs=