http urls monitor.

load.go 9.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. // Copyright 2018 Frank Schroeder. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package properties
  5. import (
  6. "fmt"
  7. "io/ioutil"
  8. "net/http"
  9. "os"
  10. "strings"
  11. )
  12. // Encoding specifies encoding of the input data.
  13. type Encoding uint
  14. const (
  15. // utf8Default is a private placeholder for the zero value of Encoding to
  16. // ensure that it has the correct meaning. UTF8 is the default encoding but
  17. // was assigned a non-zero value which cannot be changed without breaking
  18. // existing code. Clients should continue to use the public constants.
  19. utf8Default Encoding = iota
  20. // UTF8 interprets the input data as UTF-8.
  21. UTF8
  22. // ISO_8859_1 interprets the input data as ISO-8859-1.
  23. ISO_8859_1
  24. )
  25. type Loader struct {
  26. // Encoding determines how the data from files and byte buffers
  27. // is interpreted. For URLs the Content-Type header is used
  28. // to determine the encoding of the data.
  29. Encoding Encoding
  30. // DisableExpansion configures the property expansion of the
  31. // returned property object. When set to true, the property values
  32. // will not be expanded and the Property object will not be checked
  33. // for invalid expansion expressions.
  34. DisableExpansion bool
  35. // IgnoreMissing configures whether missing files or URLs which return
  36. // 404 are reported as errors. When set to true, missing files and 404
  37. // status codes are not reported as errors.
  38. IgnoreMissing bool
  39. }
  40. // Load reads a buffer into a Properties struct.
  41. func (l *Loader) LoadBytes(buf []byte) (*Properties, error) {
  42. return l.loadBytes(buf, l.Encoding)
  43. }
  44. // LoadAll reads the content of multiple URLs or files in the given order into
  45. // a Properties struct. If IgnoreMissing is true then a 404 status code or
  46. // missing file will not be reported as error. Encoding sets the encoding for
  47. // files. For the URLs see LoadURL for the Content-Type header and the
  48. // encoding.
  49. func (l *Loader) LoadAll(names []string) (*Properties, error) {
  50. all := NewProperties()
  51. for _, name := range names {
  52. n, err := expandName(name)
  53. if err != nil {
  54. return nil, err
  55. }
  56. var p *Properties
  57. switch {
  58. case strings.HasPrefix(n, "http://"):
  59. p, err = l.LoadURL(n)
  60. case strings.HasPrefix(n, "https://"):
  61. p, err = l.LoadURL(n)
  62. default:
  63. p, err = l.LoadFile(n)
  64. }
  65. if err != nil {
  66. return nil, err
  67. }
  68. all.Merge(p)
  69. }
  70. all.DisableExpansion = l.DisableExpansion
  71. if all.DisableExpansion {
  72. return all, nil
  73. }
  74. return all, all.check()
  75. }
  76. // LoadFile reads a file into a Properties struct.
  77. // If IgnoreMissing is true then a missing file will not be
  78. // reported as error.
  79. func (l *Loader) LoadFile(filename string) (*Properties, error) {
  80. data, err := ioutil.ReadFile(filename)
  81. if err != nil {
  82. if l.IgnoreMissing && os.IsNotExist(err) {
  83. LogPrintf("properties: %s not found. skipping", filename)
  84. return NewProperties(), nil
  85. }
  86. return nil, err
  87. }
  88. return l.loadBytes(data, l.Encoding)
  89. }
  90. // LoadURL reads the content of the URL into a Properties struct.
  91. //
  92. // The encoding is determined via the Content-Type header which
  93. // should be set to 'text/plain'. If the 'charset' parameter is
  94. // missing, 'iso-8859-1' or 'latin1' the encoding is set to
  95. // ISO-8859-1. If the 'charset' parameter is set to 'utf-8' the
  96. // encoding is set to UTF-8. A missing content type header is
  97. // interpreted as 'text/plain; charset=utf-8'.
  98. func (l *Loader) LoadURL(url string) (*Properties, error) {
  99. resp, err := http.Get(url)
  100. if err != nil {
  101. return nil, fmt.Errorf("properties: error fetching %q. %s", url, err)
  102. }
  103. if resp.StatusCode == 404 && l.IgnoreMissing {
  104. LogPrintf("properties: %s returned %d. skipping", url, resp.StatusCode)
  105. return NewProperties(), nil
  106. }
  107. if resp.StatusCode != 200 {
  108. return nil, fmt.Errorf("properties: %s returned %d", url, resp.StatusCode)
  109. }
  110. body, err := ioutil.ReadAll(resp.Body)
  111. if err != nil {
  112. return nil, fmt.Errorf("properties: %s error reading response. %s", url, err)
  113. }
  114. defer resp.Body.Close()
  115. ct := resp.Header.Get("Content-Type")
  116. var enc Encoding
  117. switch strings.ToLower(ct) {
  118. case "text/plain", "text/plain; charset=iso-8859-1", "text/plain; charset=latin1":
  119. enc = ISO_8859_1
  120. case "", "text/plain; charset=utf-8":
  121. enc = UTF8
  122. default:
  123. return nil, fmt.Errorf("properties: invalid content type %s", ct)
  124. }
  125. return l.loadBytes(body, enc)
  126. }
  127. func (l *Loader) loadBytes(buf []byte, enc Encoding) (*Properties, error) {
  128. p, err := parse(convert(buf, enc))
  129. if err != nil {
  130. return nil, err
  131. }
  132. p.DisableExpansion = l.DisableExpansion
  133. if p.DisableExpansion {
  134. return p, nil
  135. }
  136. return p, p.check()
  137. }
  138. // Load reads a buffer into a Properties struct.
  139. func Load(buf []byte, enc Encoding) (*Properties, error) {
  140. l := &Loader{Encoding: enc}
  141. return l.LoadBytes(buf)
  142. }
  143. // LoadString reads an UTF8 string into a properties struct.
  144. func LoadString(s string) (*Properties, error) {
  145. l := &Loader{Encoding: UTF8}
  146. return l.LoadBytes([]byte(s))
  147. }
  148. // LoadMap creates a new Properties struct from a string map.
  149. func LoadMap(m map[string]string) *Properties {
  150. p := NewProperties()
  151. for k, v := range m {
  152. p.Set(k, v)
  153. }
  154. return p
  155. }
  156. // LoadFile reads a file into a Properties struct.
  157. func LoadFile(filename string, enc Encoding) (*Properties, error) {
  158. l := &Loader{Encoding: enc}
  159. return l.LoadAll([]string{filename})
  160. }
  161. // LoadFiles reads multiple files in the given order into
  162. // a Properties struct. If 'ignoreMissing' is true then
  163. // non-existent files will not be reported as error.
  164. func LoadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) {
  165. l := &Loader{Encoding: enc, IgnoreMissing: ignoreMissing}
  166. return l.LoadAll(filenames)
  167. }
  168. // LoadURL reads the content of the URL into a Properties struct.
  169. // See Loader#LoadURL for details.
  170. func LoadURL(url string) (*Properties, error) {
  171. l := &Loader{Encoding: UTF8}
  172. return l.LoadAll([]string{url})
  173. }
  174. // LoadURLs reads the content of multiple URLs in the given order into a
  175. // Properties struct. If IgnoreMissing is true then a 404 status code will
  176. // not be reported as error. See Loader#LoadURL for the Content-Type header
  177. // and the encoding.
  178. func LoadURLs(urls []string, ignoreMissing bool) (*Properties, error) {
  179. l := &Loader{Encoding: UTF8, IgnoreMissing: ignoreMissing}
  180. return l.LoadAll(urls)
  181. }
  182. // LoadAll reads the content of multiple URLs or files in the given order into a
  183. // Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will
  184. // not be reported as error. Encoding sets the encoding for files. For the URLs please see
  185. // LoadURL for the Content-Type header and the encoding.
  186. func LoadAll(names []string, enc Encoding, ignoreMissing bool) (*Properties, error) {
  187. l := &Loader{Encoding: enc, IgnoreMissing: ignoreMissing}
  188. return l.LoadAll(names)
  189. }
  190. // MustLoadString reads an UTF8 string into a Properties struct and
  191. // panics on error.
  192. func MustLoadString(s string) *Properties {
  193. return must(LoadString(s))
  194. }
  195. // MustLoadFile reads a file into a Properties struct and
  196. // panics on error.
  197. func MustLoadFile(filename string, enc Encoding) *Properties {
  198. return must(LoadFile(filename, enc))
  199. }
  200. // MustLoadFiles reads multiple files in the given order into
  201. // a Properties struct and panics on error. If 'ignoreMissing'
  202. // is true then non-existent files will not be reported as error.
  203. func MustLoadFiles(filenames []string, enc Encoding, ignoreMissing bool) *Properties {
  204. return must(LoadFiles(filenames, enc, ignoreMissing))
  205. }
  206. // MustLoadURL reads the content of a URL into a Properties struct and
  207. // panics on error.
  208. func MustLoadURL(url string) *Properties {
  209. return must(LoadURL(url))
  210. }
  211. // MustLoadURLs reads the content of multiple URLs in the given order into a
  212. // Properties struct and panics on error. If 'ignoreMissing' is true then a 404
  213. // status code will not be reported as error.
  214. func MustLoadURLs(urls []string, ignoreMissing bool) *Properties {
  215. return must(LoadURLs(urls, ignoreMissing))
  216. }
  217. // MustLoadAll reads the content of multiple URLs or files in the given order into a
  218. // Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will
  219. // not be reported as error. Encoding sets the encoding for files. For the URLs please see
  220. // LoadURL for the Content-Type header and the encoding. It panics on error.
  221. func MustLoadAll(names []string, enc Encoding, ignoreMissing bool) *Properties {
  222. return must(LoadAll(names, enc, ignoreMissing))
  223. }
  224. func must(p *Properties, err error) *Properties {
  225. if err != nil {
  226. ErrorHandler(err)
  227. }
  228. return p
  229. }
  230. // expandName expands ${ENV_VAR} expressions in a name.
  231. // If the environment variable does not exist then it will be replaced
  232. // with an empty string. Malformed expressions like "${ENV_VAR" will
  233. // be reported as error.
  234. func expandName(name string) (string, error) {
  235. return expand(name, []string{}, "${", "}", make(map[string]string))
  236. }
  237. // Interprets a byte buffer either as an ISO-8859-1 or UTF-8 encoded string.
  238. // For ISO-8859-1 we can convert each byte straight into a rune since the
  239. // first 256 unicode code points cover ISO-8859-1.
  240. func convert(buf []byte, enc Encoding) string {
  241. switch enc {
  242. case utf8Default, UTF8:
  243. return string(buf)
  244. case ISO_8859_1:
  245. runes := make([]rune, len(buf))
  246. for i, b := range buf {
  247. runes[i] = rune(b)
  248. }
  249. return string(runes)
  250. default:
  251. ErrorHandler(fmt.Errorf("unsupported encoding %v", enc))
  252. }
  253. panic("ErrorHandler should exit")
  254. }