Source File
decode.go
Belonging Package
gopkg.in/yaml.v2
package yaml
import (
)
const (
documentNode = 1 << iota
mappingNode
sequenceNode
scalarNode
aliasNode
)
type node struct {
kind int
line, column int
For an alias node, alias holds the resolved alias.
---------------------------------------------------------------------------- Parser, produces a node tree out of a libyaml event stream.
type parser struct {
parser yaml_parser_t
event yaml_event_t
doc *node
doneInit bool
}
func ( []byte) *parser {
:= parser{}
if !yaml_parser_initialize(&.parser) {
panic("failed to initialize YAML emitter")
}
if len() == 0 {
= []byte{'\n'}
}
yaml_parser_set_input_string(&.parser, )
return &
}
func ( io.Reader) *parser {
:= parser{}
if !yaml_parser_initialize(&.parser) {
panic("failed to initialize YAML emitter")
}
yaml_parser_set_input_reader(&.parser, )
return &
}
func ( *parser) () {
if .doneInit {
return
}
.expect(yaml_STREAM_START_EVENT)
.doneInit = true
}
func ( *parser) () {
if .event.typ != yaml_NO_EVENT {
yaml_event_delete(&.event)
}
yaml_parser_delete(&.parser)
}
expect consumes an event from the event stream and checks that it's of the expected type.
func ( *parser) ( yaml_event_type_t) {
if .event.typ == yaml_NO_EVENT {
if !yaml_parser_parse(&.parser, &.event) {
.fail()
}
}
if .event.typ == yaml_STREAM_END_EVENT {
failf("attempted to go past the end of stream; corrupted value?")
}
if .event.typ != {
.parser.problem = fmt.Sprintf("expected %s event but got %s", , .event.typ)
.fail()
}
yaml_event_delete(&.event)
.event.typ = yaml_NO_EVENT
}
peek peeks at the next event in the event stream, puts the results into p.event and returns the event type.
func ( *parser) () yaml_event_type_t {
if .event.typ != yaml_NO_EVENT {
return .event.typ
}
if !yaml_parser_parse(&.parser, &.event) {
.fail()
}
return .event.typ
}
func ( *parser) () {
var string
var int
if .parser.problem_mark.line != 0 {
Scanner errors don't iterate line before returning error
if .parser.error == yaml_SCANNER_ERROR {
++
}
} else if .parser.context_mark.line != 0 {
= .parser.context_mark.line
}
if != 0 {
= "line " + strconv.Itoa() + ": "
}
var string
if len(.parser.problem) > 0 {
= .parser.problem
} else {
= "unknown problem parsing YAML content"
}
failf("%s%s", , )
}
func ( *parser) ( *node, []byte) {
if != nil {
.doc.anchors[string()] =
}
}
func ( *parser) () *node {
.init()
switch .peek() {
case yaml_SCALAR_EVENT:
return .scalar()
case yaml_ALIAS_EVENT:
return .alias()
case yaml_MAPPING_START_EVENT:
return .mapping()
case yaml_SEQUENCE_START_EVENT:
return .sequence()
case yaml_DOCUMENT_START_EVENT:
return .document()
Happens when attempting to decode an empty buffer.
return nil
default:
panic("attempted to parse unknown event: " + .event.typ.String())
}
}
func ( *parser) ( int) *node {
return &node{
kind: ,
line: .event.start_mark.line,
column: .event.start_mark.column,
}
}
func ( *parser) () *node {
:= .node(documentNode)
.anchors = make(map[string]*node)
.doc =
.expect(yaml_DOCUMENT_START_EVENT)
.children = append(.children, .parse())
.expect(yaml_DOCUMENT_END_EVENT)
return
}
func ( *parser) () *node {
:= .node(aliasNode)
.value = string(.event.anchor)
.alias = .doc.anchors[.value]
if .alias == nil {
failf("unknown anchor '%s' referenced", .value)
}
.expect(yaml_ALIAS_EVENT)
return
}
func ( *parser) () *node {
:= .node(scalarNode)
.value = string(.event.value)
.tag = string(.event.tag)
.implicit = .event.implicit
.anchor(, .event.anchor)
.expect(yaml_SCALAR_EVENT)
return
}
func ( *parser) () *node {
:= .node(sequenceNode)
.anchor(, .event.anchor)
.expect(yaml_SEQUENCE_START_EVENT)
for .peek() != yaml_SEQUENCE_END_EVENT {
.children = append(.children, .parse())
}
.expect(yaml_SEQUENCE_END_EVENT)
return
}
func ( *parser) () *node {
:= .node(mappingNode)
.anchor(, .event.anchor)
.expect(yaml_MAPPING_START_EVENT)
for .peek() != yaml_MAPPING_END_EVENT {
.children = append(.children, .parse(), .parse())
}
.expect(yaml_MAPPING_END_EVENT)
return
}
---------------------------------------------------------------------------- Decoder, unmarshals a node into a provided value.
type decoder struct {
doc *node
aliases map[*node]bool
mapType reflect.Type
terrors []string
strict bool
decodeCount int
aliasCount int
aliasDepth int
}
var (
mapItemType = reflect.TypeOf(MapItem{})
durationType = reflect.TypeOf(time.Duration(0))
defaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
ifaceType = defaultMapType.Elem()
timeType = reflect.TypeOf(time.Time{})
ptrTimeType = reflect.TypeOf(&time.Time{})
)
func ( bool) *decoder {
:= &decoder{mapType: defaultMapType, strict: }
.aliases = make(map[*node]bool)
return
}
func ( *decoder) ( *node, string, reflect.Value) {
if .tag != "" {
= .tag
}
:= .value
if != yaml_SEQ_TAG && != yaml_MAP_TAG {
if len() > 10 {
= " `" + [:7] + "...`"
} else {
= " `" + + "`"
}
}
.terrors = append(.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", .line+1, shortTag(), , .Type()))
}
func ( *decoder) ( *node, Unmarshaler) ( bool) {
:= len(.terrors)
:= .UnmarshalYAML(func( interface{}) ( error) {
defer handleErr(&)
.unmarshal(, reflect.ValueOf())
if len(.terrors) > {
:= .terrors[:]
.terrors = .terrors[:]
return &TypeError{}
}
return nil
})
if , := .(*TypeError); {
.terrors = append(.terrors, .Errors...)
return false
}
if != nil {
fail()
}
return true
}
d.prepare initializes and dereferences pointers and calls UnmarshalYAML if a value is found to implement it. It returns the initialized and dereferenced out value, whether unmarshalling was already done by UnmarshalYAML, and if so whether its types unmarshalled appropriately. If n holds a null value, prepare returns before doing anything.
func ( *decoder) ( *node, reflect.Value) ( reflect.Value, , bool) {
if .tag == yaml_NULL_TAG || .kind == scalarNode && .tag == "" && (.value == "null" || .value == "~" || .value == "" && .implicit) {
return , false, false
}
:= true
for {
= false
if .Kind() == reflect.Ptr {
if .IsNil() {
.Set(reflect.New(.Type().Elem()))
}
= .Elem()
= true
}
if .CanAddr() {
if , := .Addr().Interface().(Unmarshaler); {
= .callUnmarshaler(, )
return , true,
}
}
}
return , false, false
}
400,000 decode operations is ~500kb of dense object declarations, or ~5kb of dense object declarations with 10000% alias expansion
alias_ratio_range_low = 400000
4,000,000 decode operations is ~5MB of dense object declarations, or ~4.5MB of dense object declarations with 10% alias expansion
alias_ratio_range_high = 4000000
alias_ratio_range is the range over which we scale allowed alias ratios
alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low)
)
func ( int) float64 {
switch {
allow 99% to come from alias expansion for small-to-medium documents
return 0.99
allow 10% to come from alias expansion for very large documents
return 0.10
scale smoothly from 99% down to 10% over the range. this maps to 396,000 - 400,000 allowed alias-driven decodes over the range. 400,000 decode operations is ~100MB of allocations in worst-case scenarios (single-item maps).
return 0.99 - 0.89*(float64(-alias_ratio_range_low)/alias_ratio_range)
}
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
.decodeCount++
if .aliasDepth > 0 {
.aliasCount++
}
if .aliasCount > 100 && .decodeCount > 1000 && float64(.aliasCount)/float64(.decodeCount) > allowedAliasRatio(.decodeCount) {
failf("document contains excessive aliasing")
}
switch .kind {
case documentNode:
return .document(, )
case aliasNode:
return .alias(, )
}
, , := .prepare(, )
if {
return
}
switch .kind {
case scalarNode:
= .scalar(, )
case mappingNode:
= .mapping(, )
case sequenceNode:
= .sequence(, )
default:
panic("internal error: unknown node kind: " + strconv.Itoa(.kind))
}
return
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
if len(.children) == 1 {
.doc =
.unmarshal(.children[0], )
return true
}
return false
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
TODO this could actually be allowed in some circumstances.
failf("anchor '%s' value contains itself", .value)
}
.aliases[] = true
.aliasDepth++
= .unmarshal(.alias, )
.aliasDepth--
delete(.aliases, )
return
}
var zeroValue reflect.Value
func ( reflect.Value) {
for , := range .MapKeys() {
.SetMapIndex(, zeroValue)
}
}
func ( *decoder) ( *node, reflect.Value) bool {
var string
var interface{}
if .tag == "" && !.implicit {
= yaml_STR_TAG
= .value
} else {
, = resolve(.tag, .value)
if == yaml_BINARY_TAG {
, := base64.StdEncoding.DecodeString(.(string))
if != nil {
failf("!!binary value contains invalid base64 data")
}
= string()
}
}
if == nil {
if .Kind() == reflect.Map && !.CanAddr() {
resetMap()
} else {
.Set(reflect.Zero(.Type()))
}
return true
}
Perhaps we can use the value as a TextUnmarshaler to set its value.
if .CanAddr() {
, := .Addr().Interface().(encoding.TextUnmarshaler)
if {
var []byte
if == yaml_BINARY_TAG {
= []byte(.(string))
We let any value be unmarshaled into TextUnmarshaler. That might be more lax than we'd like, but the TextUnmarshaler itself should bowl out any dubious values.
It looks like a timestamp but for backward compatibility reasons we set it as a string, so that code that unmarshals timestamp-like values into interface{} will continue to see a string and not a time.Time. TODO(v3) Drop this.
.Set(reflect.ValueOf(.value))
} else {
.Set(reflect.ValueOf())
}
return true
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
switch resolved := .(type) {
case int:
if !.OverflowInt(int64()) {
.SetInt(int64())
return true
}
case int64:
if !.OverflowInt() {
.SetInt()
return true
}
case uint64:
if <= math.MaxInt64 && !.OverflowInt(int64()) {
.SetInt(int64())
return true
}
case float64:
if <= math.MaxInt64 && !.OverflowInt(int64()) {
.SetInt(int64())
return true
}
case string:
if .Type() == durationType {
, := time.ParseDuration()
if == nil {
.SetInt(int64())
return true
}
}
}
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
switch resolved := .(type) {
case int:
if >= 0 && !.OverflowUint(uint64()) {
.SetUint(uint64())
return true
}
case int64:
if >= 0 && !.OverflowUint(uint64()) {
.SetUint(uint64())
return true
}
case uint64:
if !.OverflowUint(uint64()) {
.SetUint(uint64())
return true
}
case float64:
if <= math.MaxUint64 && !.OverflowUint(uint64()) {
.SetUint(uint64())
return true
}
}
case reflect.Bool:
switch resolved := .(type) {
case bool:
.SetBool()
return true
}
case reflect.Float32, reflect.Float64:
switch resolved := .(type) {
case int:
.SetFloat(float64())
return true
case int64:
.SetFloat(float64())
return true
case uint64:
.SetFloat(float64())
return true
case float64:
.SetFloat()
return true
}
case reflect.Struct:
if := reflect.ValueOf(); .Type() == .Type() {
.Set()
return true
}
case reflect.Ptr:
TODO DOes this make sense? When is out a Ptr except when decoding a nil value?
:= reflect.New(.Type().Elem())
.Elem().Set(reflect.ValueOf())
.Set()
return true
}
}
.terror(, , )
return false
}
func ( interface{}) reflect.Value {
:= reflect.ValueOf()
:= reflect.New(.Type()).Elem()
.Set()
return
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
:= len(.children)
var reflect.Value
switch .Kind() {
case reflect.Slice:
.Set(reflect.MakeSlice(.Type(), , ))
case reflect.Array:
if != .Len() {
failf("invalid array: want %d elements but got %d", .Len(), )
}
No type hints. Will have to use a generic sequence.
=
= settableValueOf(make([]interface{}, ))
default:
.terror(, yaml_SEQ_TAG, )
return false
}
:= .Type().Elem()
:= 0
for := 0; < ; ++ {
:= reflect.New().Elem()
if := .unmarshal(.children[], ); {
.Index().Set()
++
}
}
if .Kind() != reflect.Array {
.Set(.Slice(0, ))
}
if .IsValid() {
.Set()
}
return true
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
switch .Kind() {
case reflect.Struct:
return .mappingStruct(, )
case reflect.Slice:
return .mappingSlice(, )
okay
case reflect.Interface:
if .mapType.Kind() == reflect.Map {
:=
= reflect.MakeMap(.mapType)
.Set()
} else {
:= reflect.New(.mapType).Elem()
if !.mappingSlice(, ) {
return false
}
.Set()
return true
}
default:
.terror(, yaml_MAP_TAG, )
return false
}
:= .Type()
:= .Key()
:= .Elem()
:= .mapType
if .Key() == ifaceType && .Elem() == ifaceType {
.mapType =
}
if .IsNil() {
.Set(reflect.MakeMap())
}
:= len(.children)
for := 0; < ; += 2 {
if isMerge(.children[]) {
.merge(.children[+1], )
continue
}
:= reflect.New().Elem()
if .unmarshal(.children[], ) {
:= .Kind()
if == reflect.Interface {
= .Elem().Kind()
}
if == reflect.Map || == reflect.Slice {
failf("invalid map key: %#v", .Interface())
}
:= reflect.New().Elem()
if .unmarshal(.children[+1], ) {
.setMapIndex(.children[+1], , , )
}
}
}
.mapType =
return true
}
func ( *decoder) ( *node, , , reflect.Value) {
if .strict && .MapIndex() != zeroValue {
.terrors = append(.terrors, fmt.Sprintf("line %d: key %#v already set in map", .line+1, .Interface()))
return
}
.SetMapIndex(, )
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
:= .Type()
if .Elem() != mapItemType {
.terror(, yaml_MAP_TAG, )
return false
}
:= .mapType
.mapType =
var []MapItem
var = len(.children)
for := 0; < ; += 2 {
if isMerge(.children[]) {
.merge(.children[+1], )
continue
}
:= MapItem{}
:= reflect.ValueOf(&.Key).Elem()
if .unmarshal(.children[], ) {
:= reflect.ValueOf(&.Value).Elem()
if .unmarshal(.children[+1], ) {
= append(, )
}
}
}
.Set(reflect.ValueOf())
.mapType =
return true
}
func ( *decoder) ( *node, reflect.Value) ( bool) {
, := getStructInfo(.Type())
if != nil {
panic()
}
:= settableValueOf("")
:= len(.children)
var reflect.Value
var reflect.Type
if .InlineMap != -1 {
= .Field(.InlineMap)
.Set(reflect.New(.Type()).Elem())
= .Type().Elem()
}
var []bool
if .strict {
= make([]bool, len(.FieldsList))
}
for := 0; < ; += 2 {
:= .children[]
if isMerge() {
.merge(.children[+1], )
continue
}
if !.unmarshal(, ) {
continue
}
if , := .FieldsMap[.String()]; {
if .strict {
if [.Id] {
.terrors = append(.terrors, fmt.Sprintf("line %d: field %s already set in type %s", .line+1, .String(), .Type()))
continue
}
[.Id] = true
}
var reflect.Value
if .Inline == nil {
= .Field(.Num)
} else {
= .FieldByIndex(.Inline)
}
.unmarshal(.children[+1], )
} else if .InlineMap != -1 {
if .IsNil() {
.Set(reflect.MakeMap(.Type()))
}
:= reflect.New().Elem()
.unmarshal(.children[+1], )
.setMapIndex(.children[+1], , , )
} else if .strict {
.terrors = append(.terrors, fmt.Sprintf("line %d: field %s not found in type %s", .line+1, .String(), .Type()))
}
}
return true
}
func () {
failf("map merge requires map or sequence of maps as the value")
}
func ( *decoder) ( *node, reflect.Value) {
switch .kind {
case mappingNode:
.unmarshal(, )
case aliasNode:
if .alias != nil && .alias.kind != mappingNode {
failWantMap()
}
.unmarshal(, )
Step backwards as earlier nodes take precedence.
for := len(.children) - 1; >= 0; -- {
:= .children[]
if .kind == aliasNode {
if .alias != nil && .alias.kind != mappingNode {
failWantMap()
}
} else if .kind != mappingNode {
failWantMap()
}
.unmarshal(, )
}
default:
failWantMap()
}
}
func ( *node) bool {
return .kind == scalarNode && .value == "<<" && (.implicit == true || .tag == yaml_MERGE_TAG)
 |
The pages are generated with Golds v0.3.2-preview. (GOOS=darwin GOARCH=amd64) Golds is a Go 101 project developed by Tapir Liu. PR and bug reports are welcome and can be submitted to the issue list. Please follow @Go100and1 (reachable from the left QR code) to get the latest news of Golds. |