http urls monitor.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730
  1. package hcl
  2. import (
  3. "errors"
  4. "fmt"
  5. "reflect"
  6. "sort"
  7. "strconv"
  8. "strings"
  9. "github.com/hashicorp/hcl/hcl/ast"
  10. "github.com/hashicorp/hcl/hcl/parser"
  11. "github.com/hashicorp/hcl/hcl/token"
  12. )
  13. // This is the tag to use with structures to have settings for HCL
  14. const tagName = "hcl"
  15. var (
  16. // nodeType holds a reference to the type of ast.Node
  17. nodeType reflect.Type = findNodeType()
  18. )
  19. // Unmarshal accepts a byte slice as input and writes the
  20. // data to the value pointed to by v.
  21. func Unmarshal(bs []byte, v interface{}) error {
  22. root, err := parse(bs)
  23. if err != nil {
  24. return err
  25. }
  26. return DecodeObject(v, root)
  27. }
  28. // Decode reads the given input and decodes it into the structure
  29. // given by `out`.
  30. func Decode(out interface{}, in string) error {
  31. obj, err := Parse(in)
  32. if err != nil {
  33. return err
  34. }
  35. return DecodeObject(out, obj)
  36. }
  37. // DecodeObject is a lower-level version of Decode. It decodes a
  38. // raw Object into the given output.
  39. func DecodeObject(out interface{}, n ast.Node) error {
  40. val := reflect.ValueOf(out)
  41. if val.Kind() != reflect.Ptr {
  42. return errors.New("result must be a pointer")
  43. }
  44. // If we have the file, we really decode the root node
  45. if f, ok := n.(*ast.File); ok {
  46. n = f.Node
  47. }
  48. var d decoder
  49. return d.decode("root", n, val.Elem())
  50. }
  51. type decoder struct {
  52. stack []reflect.Kind
  53. }
  54. func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error {
  55. k := result
  56. // If we have an interface with a valid value, we use that
  57. // for the check.
  58. if result.Kind() == reflect.Interface {
  59. elem := result.Elem()
  60. if elem.IsValid() {
  61. k = elem
  62. }
  63. }
  64. // Push current onto stack unless it is an interface.
  65. if k.Kind() != reflect.Interface {
  66. d.stack = append(d.stack, k.Kind())
  67. // Schedule a pop
  68. defer func() {
  69. d.stack = d.stack[:len(d.stack)-1]
  70. }()
  71. }
  72. switch k.Kind() {
  73. case reflect.Bool:
  74. return d.decodeBool(name, node, result)
  75. case reflect.Float32, reflect.Float64:
  76. return d.decodeFloat(name, node, result)
  77. case reflect.Int, reflect.Int32, reflect.Int64:
  78. return d.decodeInt(name, node, result)
  79. case reflect.Interface:
  80. // When we see an interface, we make our own thing
  81. return d.decodeInterface(name, node, result)
  82. case reflect.Map:
  83. return d.decodeMap(name, node, result)
  84. case reflect.Ptr:
  85. return d.decodePtr(name, node, result)
  86. case reflect.Slice:
  87. return d.decodeSlice(name, node, result)
  88. case reflect.String:
  89. return d.decodeString(name, node, result)
  90. case reflect.Struct:
  91. return d.decodeStruct(name, node, result)
  92. default:
  93. return &parser.PosError{
  94. Pos: node.Pos(),
  95. Err: fmt.Errorf("%s: unknown kind to decode into: %s", name, k.Kind()),
  96. }
  97. }
  98. }
  99. func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) error {
  100. switch n := node.(type) {
  101. case *ast.LiteralType:
  102. if n.Token.Type == token.BOOL {
  103. v, err := strconv.ParseBool(n.Token.Text)
  104. if err != nil {
  105. return err
  106. }
  107. result.Set(reflect.ValueOf(v))
  108. return nil
  109. }
  110. }
  111. return &parser.PosError{
  112. Pos: node.Pos(),
  113. Err: fmt.Errorf("%s: unknown type %T", name, node),
  114. }
  115. }
  116. func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error {
  117. switch n := node.(type) {
  118. case *ast.LiteralType:
  119. if n.Token.Type == token.FLOAT || n.Token.Type == token.NUMBER {
  120. v, err := strconv.ParseFloat(n.Token.Text, 64)
  121. if err != nil {
  122. return err
  123. }
  124. result.Set(reflect.ValueOf(v).Convert(result.Type()))
  125. return nil
  126. }
  127. }
  128. return &parser.PosError{
  129. Pos: node.Pos(),
  130. Err: fmt.Errorf("%s: unknown type %T", name, node),
  131. }
  132. }
  133. func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) error {
  134. switch n := node.(type) {
  135. case *ast.LiteralType:
  136. switch n.Token.Type {
  137. case token.NUMBER:
  138. v, err := strconv.ParseInt(n.Token.Text, 0, 0)
  139. if err != nil {
  140. return err
  141. }
  142. if result.Kind() == reflect.Interface {
  143. result.Set(reflect.ValueOf(int(v)))
  144. } else {
  145. result.SetInt(v)
  146. }
  147. return nil
  148. case token.STRING:
  149. v, err := strconv.ParseInt(n.Token.Value().(string), 0, 0)
  150. if err != nil {
  151. return err
  152. }
  153. if result.Kind() == reflect.Interface {
  154. result.Set(reflect.ValueOf(int(v)))
  155. } else {
  156. result.SetInt(v)
  157. }
  158. return nil
  159. }
  160. }
  161. return &parser.PosError{
  162. Pos: node.Pos(),
  163. Err: fmt.Errorf("%s: unknown type %T", name, node),
  164. }
  165. }
  166. func (d *decoder) decodeInterface(name string, node ast.Node, result reflect.Value) error {
  167. // When we see an ast.Node, we retain the value to enable deferred decoding.
  168. // Very useful in situations where we want to preserve ast.Node information
  169. // like Pos
  170. if result.Type() == nodeType && result.CanSet() {
  171. result.Set(reflect.ValueOf(node))
  172. return nil
  173. }
  174. var set reflect.Value
  175. redecode := true
  176. // For testing types, ObjectType should just be treated as a list. We
  177. // set this to a temporary var because we want to pass in the real node.
  178. testNode := node
  179. if ot, ok := node.(*ast.ObjectType); ok {
  180. testNode = ot.List
  181. }
  182. switch n := testNode.(type) {
  183. case *ast.ObjectList:
  184. // If we're at the root or we're directly within a slice, then we
  185. // decode objects into map[string]interface{}, otherwise we decode
  186. // them into lists.
  187. if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice {
  188. var temp map[string]interface{}
  189. tempVal := reflect.ValueOf(temp)
  190. result := reflect.MakeMap(
  191. reflect.MapOf(
  192. reflect.TypeOf(""),
  193. tempVal.Type().Elem()))
  194. set = result
  195. } else {
  196. var temp []map[string]interface{}
  197. tempVal := reflect.ValueOf(temp)
  198. result := reflect.MakeSlice(
  199. reflect.SliceOf(tempVal.Type().Elem()), 0, len(n.Items))
  200. set = result
  201. }
  202. case *ast.ObjectType:
  203. // If we're at the root or we're directly within a slice, then we
  204. // decode objects into map[string]interface{}, otherwise we decode
  205. // them into lists.
  206. if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice {
  207. var temp map[string]interface{}
  208. tempVal := reflect.ValueOf(temp)
  209. result := reflect.MakeMap(
  210. reflect.MapOf(
  211. reflect.TypeOf(""),
  212. tempVal.Type().Elem()))
  213. set = result
  214. } else {
  215. var temp []map[string]interface{}
  216. tempVal := reflect.ValueOf(temp)
  217. result := reflect.MakeSlice(
  218. reflect.SliceOf(tempVal.Type().Elem()), 0, 1)
  219. set = result
  220. }
  221. case *ast.ListType:
  222. var temp []interface{}
  223. tempVal := reflect.ValueOf(temp)
  224. result := reflect.MakeSlice(
  225. reflect.SliceOf(tempVal.Type().Elem()), 0, 0)
  226. set = result
  227. case *ast.LiteralType:
  228. switch n.Token.Type {
  229. case token.BOOL:
  230. var result bool
  231. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  232. case token.FLOAT:
  233. var result float64
  234. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  235. case token.NUMBER:
  236. var result int
  237. set = reflect.Indirect(reflect.New(reflect.TypeOf(result)))
  238. case token.STRING, token.HEREDOC:
  239. set = reflect.Indirect(reflect.New(reflect.TypeOf("")))
  240. default:
  241. return &parser.PosError{
  242. Pos: node.Pos(),
  243. Err: fmt.Errorf("%s: cannot decode into interface: %T", name, node),
  244. }
  245. }
  246. default:
  247. return fmt.Errorf(
  248. "%s: cannot decode into interface: %T",
  249. name, node)
  250. }
  251. // Set the result to what its supposed to be, then reset
  252. // result so we don't reflect into this method anymore.
  253. result.Set(set)
  254. if redecode {
  255. // Revisit the node so that we can use the newly instantiated
  256. // thing and populate it.
  257. if err := d.decode(name, node, result); err != nil {
  258. return err
  259. }
  260. }
  261. return nil
  262. }
  263. func (d *decoder) decodeMap(name string, node ast.Node, result reflect.Value) error {
  264. if item, ok := node.(*ast.ObjectItem); ok {
  265. node = &ast.ObjectList{Items: []*ast.ObjectItem{item}}
  266. }
  267. if ot, ok := node.(*ast.ObjectType); ok {
  268. node = ot.List
  269. }
  270. n, ok := node.(*ast.ObjectList)
  271. if !ok {
  272. return &parser.PosError{
  273. Pos: node.Pos(),
  274. Err: fmt.Errorf("%s: not an object type for map (%T)", name, node),
  275. }
  276. }
  277. // If we have an interface, then we can address the interface,
  278. // but not the slice itself, so get the element but set the interface
  279. set := result
  280. if result.Kind() == reflect.Interface {
  281. result = result.Elem()
  282. }
  283. resultType := result.Type()
  284. resultElemType := resultType.Elem()
  285. resultKeyType := resultType.Key()
  286. if resultKeyType.Kind() != reflect.String {
  287. return &parser.PosError{
  288. Pos: node.Pos(),
  289. Err: fmt.Errorf("%s: map must have string keys", name),
  290. }
  291. }
  292. // Make a map if it is nil
  293. resultMap := result
  294. if result.IsNil() {
  295. resultMap = reflect.MakeMap(
  296. reflect.MapOf(resultKeyType, resultElemType))
  297. }
  298. // Go through each element and decode it.
  299. done := make(map[string]struct{})
  300. for _, item := range n.Items {
  301. if item.Val == nil {
  302. continue
  303. }
  304. // github.com/hashicorp/terraform/issue/5740
  305. if len(item.Keys) == 0 {
  306. return &parser.PosError{
  307. Pos: node.Pos(),
  308. Err: fmt.Errorf("%s: map must have string keys", name),
  309. }
  310. }
  311. // Get the key we're dealing with, which is the first item
  312. keyStr := item.Keys[0].Token.Value().(string)
  313. // If we've already processed this key, then ignore it
  314. if _, ok := done[keyStr]; ok {
  315. continue
  316. }
  317. // Determine the value. If we have more than one key, then we
  318. // get the objectlist of only these keys.
  319. itemVal := item.Val
  320. if len(item.Keys) > 1 {
  321. itemVal = n.Filter(keyStr)
  322. done[keyStr] = struct{}{}
  323. }
  324. // Make the field name
  325. fieldName := fmt.Sprintf("%s.%s", name, keyStr)
  326. // Get the key/value as reflection values
  327. key := reflect.ValueOf(keyStr)
  328. val := reflect.Indirect(reflect.New(resultElemType))
  329. // If we have a pre-existing value in the map, use that
  330. oldVal := resultMap.MapIndex(key)
  331. if oldVal.IsValid() {
  332. val.Set(oldVal)
  333. }
  334. // Decode!
  335. if err := d.decode(fieldName, itemVal, val); err != nil {
  336. return err
  337. }
  338. // Set the value on the map
  339. resultMap.SetMapIndex(key, val)
  340. }
  341. // Set the final map if we can
  342. set.Set(resultMap)
  343. return nil
  344. }
  345. func (d *decoder) decodePtr(name string, node ast.Node, result reflect.Value) error {
  346. // Create an element of the concrete (non pointer) type and decode
  347. // into that. Then set the value of the pointer to this type.
  348. resultType := result.Type()
  349. resultElemType := resultType.Elem()
  350. val := reflect.New(resultElemType)
  351. if err := d.decode(name, node, reflect.Indirect(val)); err != nil {
  352. return err
  353. }
  354. result.Set(val)
  355. return nil
  356. }
  357. func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) error {
  358. // If we have an interface, then we can address the interface,
  359. // but not the slice itself, so get the element but set the interface
  360. set := result
  361. if result.Kind() == reflect.Interface {
  362. result = result.Elem()
  363. }
  364. // Create the slice if it isn't nil
  365. resultType := result.Type()
  366. resultElemType := resultType.Elem()
  367. if result.IsNil() {
  368. resultSliceType := reflect.SliceOf(resultElemType)
  369. result = reflect.MakeSlice(
  370. resultSliceType, 0, 0)
  371. }
  372. // Figure out the items we'll be copying into the slice
  373. var items []ast.Node
  374. switch n := node.(type) {
  375. case *ast.ObjectList:
  376. items = make([]ast.Node, len(n.Items))
  377. for i, item := range n.Items {
  378. items[i] = item
  379. }
  380. case *ast.ObjectType:
  381. items = []ast.Node{n}
  382. case *ast.ListType:
  383. items = n.List
  384. default:
  385. return &parser.PosError{
  386. Pos: node.Pos(),
  387. Err: fmt.Errorf("unknown slice type: %T", node),
  388. }
  389. }
  390. for i, item := range items {
  391. fieldName := fmt.Sprintf("%s[%d]", name, i)
  392. // Decode
  393. val := reflect.Indirect(reflect.New(resultElemType))
  394. // if item is an object that was decoded from ambiguous JSON and
  395. // flattened, make sure it's expanded if it needs to decode into a
  396. // defined structure.
  397. item := expandObject(item, val)
  398. if err := d.decode(fieldName, item, val); err != nil {
  399. return err
  400. }
  401. // Append it onto the slice
  402. result = reflect.Append(result, val)
  403. }
  404. set.Set(result)
  405. return nil
  406. }
  407. // expandObject detects if an ambiguous JSON object was flattened to a List which
  408. // should be decoded into a struct, and expands the ast to properly deocode.
  409. func expandObject(node ast.Node, result reflect.Value) ast.Node {
  410. item, ok := node.(*ast.ObjectItem)
  411. if !ok {
  412. return node
  413. }
  414. elemType := result.Type()
  415. // our target type must be a struct
  416. switch elemType.Kind() {
  417. case reflect.Ptr:
  418. switch elemType.Elem().Kind() {
  419. case reflect.Struct:
  420. //OK
  421. default:
  422. return node
  423. }
  424. case reflect.Struct:
  425. //OK
  426. default:
  427. return node
  428. }
  429. // A list value will have a key and field name. If it had more fields,
  430. // it wouldn't have been flattened.
  431. if len(item.Keys) != 2 {
  432. return node
  433. }
  434. keyToken := item.Keys[0].Token
  435. item.Keys = item.Keys[1:]
  436. // we need to un-flatten the ast enough to decode
  437. newNode := &ast.ObjectItem{
  438. Keys: []*ast.ObjectKey{
  439. &ast.ObjectKey{
  440. Token: keyToken,
  441. },
  442. },
  443. Val: &ast.ObjectType{
  444. List: &ast.ObjectList{
  445. Items: []*ast.ObjectItem{item},
  446. },
  447. },
  448. }
  449. return newNode
  450. }
  451. func (d *decoder) decodeString(name string, node ast.Node, result reflect.Value) error {
  452. switch n := node.(type) {
  453. case *ast.LiteralType:
  454. switch n.Token.Type {
  455. case token.NUMBER:
  456. result.Set(reflect.ValueOf(n.Token.Text).Convert(result.Type()))
  457. return nil
  458. case token.STRING, token.HEREDOC:
  459. result.Set(reflect.ValueOf(n.Token.Value()).Convert(result.Type()))
  460. return nil
  461. }
  462. }
  463. return &parser.PosError{
  464. Pos: node.Pos(),
  465. Err: fmt.Errorf("%s: unknown type for string %T", name, node),
  466. }
  467. }
  468. func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) error {
  469. var item *ast.ObjectItem
  470. if it, ok := node.(*ast.ObjectItem); ok {
  471. item = it
  472. node = it.Val
  473. }
  474. if ot, ok := node.(*ast.ObjectType); ok {
  475. node = ot.List
  476. }
  477. // Handle the special case where the object itself is a literal. Previously
  478. // the yacc parser would always ensure top-level elements were arrays. The new
  479. // parser does not make the same guarantees, thus we need to convert any
  480. // top-level literal elements into a list.
  481. if _, ok := node.(*ast.LiteralType); ok && item != nil {
  482. node = &ast.ObjectList{Items: []*ast.ObjectItem{item}}
  483. }
  484. list, ok := node.(*ast.ObjectList)
  485. if !ok {
  486. return &parser.PosError{
  487. Pos: node.Pos(),
  488. Err: fmt.Errorf("%s: not an object type for struct (%T)", name, node),
  489. }
  490. }
  491. // This slice will keep track of all the structs we'll be decoding.
  492. // There can be more than one struct if there are embedded structs
  493. // that are squashed.
  494. structs := make([]reflect.Value, 1, 5)
  495. structs[0] = result
  496. // Compile the list of all the fields that we're going to be decoding
  497. // from all the structs.
  498. type field struct {
  499. field reflect.StructField
  500. val reflect.Value
  501. }
  502. fields := []field{}
  503. for len(structs) > 0 {
  504. structVal := structs[0]
  505. structs = structs[1:]
  506. structType := structVal.Type()
  507. for i := 0; i < structType.NumField(); i++ {
  508. fieldType := structType.Field(i)
  509. tagParts := strings.Split(fieldType.Tag.Get(tagName), ",")
  510. // Ignore fields with tag name "-"
  511. if tagParts[0] == "-" {
  512. continue
  513. }
  514. if fieldType.Anonymous {
  515. fieldKind := fieldType.Type.Kind()
  516. if fieldKind != reflect.Struct {
  517. return &parser.PosError{
  518. Pos: node.Pos(),
  519. Err: fmt.Errorf("%s: unsupported type to struct: %s",
  520. fieldType.Name, fieldKind),
  521. }
  522. }
  523. // We have an embedded field. We "squash" the fields down
  524. // if specified in the tag.
  525. squash := false
  526. for _, tag := range tagParts[1:] {
  527. if tag == "squash" {
  528. squash = true
  529. break
  530. }
  531. }
  532. if squash {
  533. structs = append(
  534. structs, result.FieldByName(fieldType.Name))
  535. continue
  536. }
  537. }
  538. // Normal struct field, store it away
  539. fields = append(fields, field{fieldType, structVal.Field(i)})
  540. }
  541. }
  542. usedKeys := make(map[string]struct{})
  543. decodedFields := make([]string, 0, len(fields))
  544. decodedFieldsVal := make([]reflect.Value, 0)
  545. unusedKeysVal := make([]reflect.Value, 0)
  546. for _, f := range fields {
  547. field, fieldValue := f.field, f.val
  548. if !fieldValue.IsValid() {
  549. // This should never happen
  550. panic("field is not valid")
  551. }
  552. // If we can't set the field, then it is unexported or something,
  553. // and we just continue onwards.
  554. if !fieldValue.CanSet() {
  555. continue
  556. }
  557. fieldName := field.Name
  558. tagValue := field.Tag.Get(tagName)
  559. tagParts := strings.SplitN(tagValue, ",", 2)
  560. if len(tagParts) >= 2 {
  561. switch tagParts[1] {
  562. case "decodedFields":
  563. decodedFieldsVal = append(decodedFieldsVal, fieldValue)
  564. continue
  565. case "key":
  566. if item == nil {
  567. return &parser.PosError{
  568. Pos: node.Pos(),
  569. Err: fmt.Errorf("%s: %s asked for 'key', impossible",
  570. name, fieldName),
  571. }
  572. }
  573. fieldValue.SetString(item.Keys[0].Token.Value().(string))
  574. continue
  575. case "unusedKeys":
  576. unusedKeysVal = append(unusedKeysVal, fieldValue)
  577. continue
  578. }
  579. }
  580. if tagParts[0] != "" {
  581. fieldName = tagParts[0]
  582. }
  583. // Determine the element we'll use to decode. If it is a single
  584. // match (only object with the field), then we decode it exactly.
  585. // If it is a prefix match, then we decode the matches.
  586. filter := list.Filter(fieldName)
  587. prefixMatches := filter.Children()
  588. matches := filter.Elem()
  589. if len(matches.Items) == 0 && len(prefixMatches.Items) == 0 {
  590. continue
  591. }
  592. // Track the used key
  593. usedKeys[fieldName] = struct{}{}
  594. // Create the field name and decode. We range over the elements
  595. // because we actually want the value.
  596. fieldName = fmt.Sprintf("%s.%s", name, fieldName)
  597. if len(prefixMatches.Items) > 0 {
  598. if err := d.decode(fieldName, prefixMatches, fieldValue); err != nil {
  599. return err
  600. }
  601. }
  602. for _, match := range matches.Items {
  603. var decodeNode ast.Node = match.Val
  604. if ot, ok := decodeNode.(*ast.ObjectType); ok {
  605. decodeNode = &ast.ObjectList{Items: ot.List.Items}
  606. }
  607. if err := d.decode(fieldName, decodeNode, fieldValue); err != nil {
  608. return err
  609. }
  610. }
  611. decodedFields = append(decodedFields, field.Name)
  612. }
  613. if len(decodedFieldsVal) > 0 {
  614. // Sort it so that it is deterministic
  615. sort.Strings(decodedFields)
  616. for _, v := range decodedFieldsVal {
  617. v.Set(reflect.ValueOf(decodedFields))
  618. }
  619. }
  620. return nil
  621. }
  622. // findNodeType returns the type of ast.Node
  623. func findNodeType() reflect.Type {
  624. var nodeContainer struct {
  625. Node ast.Node
  626. }
  627. value := reflect.ValueOf(nodeContainer).FieldByName("Node")
  628. return value.Type()
  629. }