http urls monitor.

util.go 7.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331
  1. // Copyright ©2015 Steve Francia <spf@spf13.com>
  2. // Portions Copyright ©2015 The Hugo Authors
  3. // Portions Copyright 2016-present Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
  4. //
  5. // Licensed under the Apache License, Version 2.0 (the "License");
  6. // you may not use this file except in compliance with the License.
  7. // You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. package afero
  17. import (
  18. "bytes"
  19. "fmt"
  20. "io"
  21. "os"
  22. "path/filepath"
  23. "strings"
  24. "unicode"
  25. "golang.org/x/text/transform"
  26. "golang.org/x/text/unicode/norm"
  27. )
  28. // Filepath separator defined by os.Separator.
  29. const FilePathSeparator = string(filepath.Separator)
  30. // Takes a reader and a path and writes the content
  31. func (a Afero) WriteReader(path string, r io.Reader) (err error) {
  32. return WriteReader(a.Fs, path, r)
  33. }
  34. func WriteReader(fs Fs, path string, r io.Reader) (err error) {
  35. dir, _ := filepath.Split(path)
  36. ospath := filepath.FromSlash(dir)
  37. if ospath != "" {
  38. err = fs.MkdirAll(ospath, 0777) // rwx, rw, r
  39. if err != nil {
  40. if err != os.ErrExist {
  41. return err
  42. }
  43. }
  44. }
  45. file, err := fs.Create(path)
  46. if err != nil {
  47. return
  48. }
  49. defer file.Close()
  50. _, err = io.Copy(file, r)
  51. return
  52. }
  53. // Same as WriteReader but checks to see if file/directory already exists.
  54. func (a Afero) SafeWriteReader(path string, r io.Reader) (err error) {
  55. return SafeWriteReader(a.Fs, path, r)
  56. }
  57. func SafeWriteReader(fs Fs, path string, r io.Reader) (err error) {
  58. dir, _ := filepath.Split(path)
  59. ospath := filepath.FromSlash(dir)
  60. if ospath != "" {
  61. err = fs.MkdirAll(ospath, 0777) // rwx, rw, r
  62. if err != nil {
  63. return
  64. }
  65. }
  66. exists, err := Exists(fs, path)
  67. if err != nil {
  68. return
  69. }
  70. if exists {
  71. return fmt.Errorf("%v already exists", path)
  72. }
  73. file, err := fs.Create(path)
  74. if err != nil {
  75. return
  76. }
  77. defer file.Close()
  78. _, err = io.Copy(file, r)
  79. return
  80. }
  81. func (a Afero) GetTempDir(subPath string) string {
  82. return GetTempDir(a.Fs, subPath)
  83. }
  84. // GetTempDir returns the default temp directory with trailing slash
  85. // if subPath is not empty then it will be created recursively with mode 777 rwx rwx rwx
  86. func GetTempDir(fs Fs, subPath string) string {
  87. addSlash := func(p string) string {
  88. if FilePathSeparator != p[len(p)-1:] {
  89. p = p + FilePathSeparator
  90. }
  91. return p
  92. }
  93. dir := addSlash(os.TempDir())
  94. if subPath != "" {
  95. // preserve windows backslash :-(
  96. if FilePathSeparator == "\\" {
  97. subPath = strings.Replace(subPath, "\\", "____", -1)
  98. }
  99. dir = dir + UnicodeSanitize((subPath))
  100. if FilePathSeparator == "\\" {
  101. dir = strings.Replace(dir, "____", "\\", -1)
  102. }
  103. if exists, _ := Exists(fs, dir); exists {
  104. return addSlash(dir)
  105. }
  106. err := fs.MkdirAll(dir, 0777)
  107. if err != nil {
  108. panic(err)
  109. }
  110. dir = addSlash(dir)
  111. }
  112. return dir
  113. }
  114. // Rewrite string to remove non-standard path characters
  115. func UnicodeSanitize(s string) string {
  116. source := []rune(s)
  117. target := make([]rune, 0, len(source))
  118. for _, r := range source {
  119. if unicode.IsLetter(r) ||
  120. unicode.IsDigit(r) ||
  121. unicode.IsMark(r) ||
  122. r == '.' ||
  123. r == '/' ||
  124. r == '\\' ||
  125. r == '_' ||
  126. r == '-' ||
  127. r == '%' ||
  128. r == ' ' ||
  129. r == '#' {
  130. target = append(target, r)
  131. }
  132. }
  133. return string(target)
  134. }
  135. // Transform characters with accents into plain forms.
  136. func NeuterAccents(s string) string {
  137. t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
  138. result, _, _ := transform.String(t, string(s))
  139. return result
  140. }
  141. func isMn(r rune) bool {
  142. return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
  143. }
  144. func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) {
  145. return FileContainsBytes(a.Fs, filename, subslice)
  146. }
  147. // Check if a file contains a specified byte slice.
  148. func FileContainsBytes(fs Fs, filename string, subslice []byte) (bool, error) {
  149. f, err := fs.Open(filename)
  150. if err != nil {
  151. return false, err
  152. }
  153. defer f.Close()
  154. return readerContainsAny(f, subslice), nil
  155. }
  156. func (a Afero) FileContainsAnyBytes(filename string, subslices [][]byte) (bool, error) {
  157. return FileContainsAnyBytes(a.Fs, filename, subslices)
  158. }
  159. // Check if a file contains any of the specified byte slices.
  160. func FileContainsAnyBytes(fs Fs, filename string, subslices [][]byte) (bool, error) {
  161. f, err := fs.Open(filename)
  162. if err != nil {
  163. return false, err
  164. }
  165. defer f.Close()
  166. return readerContainsAny(f, subslices...), nil
  167. }
  168. // readerContains reports whether any of the subslices is within r.
  169. func readerContainsAny(r io.Reader, subslices ...[]byte) bool {
  170. if r == nil || len(subslices) == 0 {
  171. return false
  172. }
  173. largestSlice := 0
  174. for _, sl := range subslices {
  175. if len(sl) > largestSlice {
  176. largestSlice = len(sl)
  177. }
  178. }
  179. if largestSlice == 0 {
  180. return false
  181. }
  182. bufflen := largestSlice * 4
  183. halflen := bufflen / 2
  184. buff := make([]byte, bufflen)
  185. var err error
  186. var n, i int
  187. for {
  188. i++
  189. if i == 1 {
  190. n, err = io.ReadAtLeast(r, buff[:halflen], halflen)
  191. } else {
  192. if i != 2 {
  193. // shift left to catch overlapping matches
  194. copy(buff[:], buff[halflen:])
  195. }
  196. n, err = io.ReadAtLeast(r, buff[halflen:], halflen)
  197. }
  198. if n > 0 {
  199. for _, sl := range subslices {
  200. if bytes.Contains(buff, sl) {
  201. return true
  202. }
  203. }
  204. }
  205. if err != nil {
  206. break
  207. }
  208. }
  209. return false
  210. }
  211. func (a Afero) DirExists(path string) (bool, error) {
  212. return DirExists(a.Fs, path)
  213. }
  214. // DirExists checks if a path exists and is a directory.
  215. func DirExists(fs Fs, path string) (bool, error) {
  216. fi, err := fs.Stat(path)
  217. if err == nil && fi.IsDir() {
  218. return true, nil
  219. }
  220. if os.IsNotExist(err) {
  221. return false, nil
  222. }
  223. return false, err
  224. }
  225. func (a Afero) IsDir(path string) (bool, error) {
  226. return IsDir(a.Fs, path)
  227. }
  228. // IsDir checks if a given path is a directory.
  229. func IsDir(fs Fs, path string) (bool, error) {
  230. fi, err := fs.Stat(path)
  231. if err != nil {
  232. return false, err
  233. }
  234. return fi.IsDir(), nil
  235. }
  236. func (a Afero) IsEmpty(path string) (bool, error) {
  237. return IsEmpty(a.Fs, path)
  238. }
  239. // IsEmpty checks if a given file or directory is empty.
  240. func IsEmpty(fs Fs, path string) (bool, error) {
  241. if b, _ := Exists(fs, path); !b {
  242. return false, fmt.Errorf("%q path does not exist", path)
  243. }
  244. fi, err := fs.Stat(path)
  245. if err != nil {
  246. return false, err
  247. }
  248. if fi.IsDir() {
  249. f, err := fs.Open(path)
  250. if err != nil {
  251. return false, err
  252. }
  253. defer f.Close()
  254. list, err := f.Readdir(-1)
  255. return len(list) == 0, nil
  256. }
  257. return fi.Size() == 0, nil
  258. }
  259. func (a Afero) Exists(path string) (bool, error) {
  260. return Exists(a.Fs, path)
  261. }
  262. // Check if a file or directory exists.
  263. func Exists(fs Fs, path string) (bool, error) {
  264. _, err := fs.Stat(path)
  265. if err == nil {
  266. return true, nil
  267. }
  268. if os.IsNotExist(err) {
  269. return false, nil
  270. }
  271. return false, err
  272. }
  273. func FullBaseFsPath(basePathFs *BasePathFs, relativePath string) string {
  274. combinedPath := filepath.Join(basePathFs.path, relativePath)
  275. if parent, ok := basePathFs.source.(*BasePathFs); ok {
  276. return FullBaseFsPath(parent, combinedPath)
  277. }
  278. return combinedPath
  279. }