http urls monitor.

cacheOnReadFs.go 6.7KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. package afero
  2. import (
  3. "os"
  4. "syscall"
  5. "time"
  6. )
  7. // If the cache duration is 0, cache time will be unlimited, i.e. once
  8. // a file is in the layer, the base will never be read again for this file.
  9. //
  10. // For cache times greater than 0, the modification time of a file is
  11. // checked. Note that a lot of file system implementations only allow a
  12. // resolution of a second for timestamps... or as the godoc for os.Chtimes()
  13. // states: "The underlying filesystem may truncate or round the values to a
  14. // less precise time unit."
  15. //
  16. // This caching union will forward all write calls also to the base file
  17. // system first. To prevent writing to the base Fs, wrap it in a read-only
  18. // filter - Note: this will also make the overlay read-only, for writing files
  19. // in the overlay, use the overlay Fs directly, not via the union Fs.
  20. type CacheOnReadFs struct {
  21. base Fs
  22. layer Fs
  23. cacheTime time.Duration
  24. }
  25. func NewCacheOnReadFs(base Fs, layer Fs, cacheTime time.Duration) Fs {
  26. return &CacheOnReadFs{base: base, layer: layer, cacheTime: cacheTime}
  27. }
  28. type cacheState int
  29. const (
  30. // not present in the overlay, unknown if it exists in the base:
  31. cacheMiss cacheState = iota
  32. // present in the overlay and in base, base file is newer:
  33. cacheStale
  34. // present in the overlay - with cache time == 0 it may exist in the base,
  35. // with cacheTime > 0 it exists in the base and is same age or newer in the
  36. // overlay
  37. cacheHit
  38. // happens if someone writes directly to the overlay without
  39. // going through this union
  40. cacheLocal
  41. )
  42. func (u *CacheOnReadFs) cacheStatus(name string) (state cacheState, fi os.FileInfo, err error) {
  43. var lfi, bfi os.FileInfo
  44. lfi, err = u.layer.Stat(name)
  45. if err == nil {
  46. if u.cacheTime == 0 {
  47. return cacheHit, lfi, nil
  48. }
  49. if lfi.ModTime().Add(u.cacheTime).Before(time.Now()) {
  50. bfi, err = u.base.Stat(name)
  51. if err != nil {
  52. return cacheLocal, lfi, nil
  53. }
  54. if bfi.ModTime().After(lfi.ModTime()) {
  55. return cacheStale, bfi, nil
  56. }
  57. }
  58. return cacheHit, lfi, nil
  59. }
  60. if err == syscall.ENOENT || os.IsNotExist(err) {
  61. return cacheMiss, nil, nil
  62. }
  63. return cacheMiss, nil, err
  64. }
  65. func (u *CacheOnReadFs) copyToLayer(name string) error {
  66. return copyToLayer(u.base, u.layer, name)
  67. }
  68. func (u *CacheOnReadFs) Chtimes(name string, atime, mtime time.Time) error {
  69. st, _, err := u.cacheStatus(name)
  70. if err != nil {
  71. return err
  72. }
  73. switch st {
  74. case cacheLocal:
  75. case cacheHit:
  76. err = u.base.Chtimes(name, atime, mtime)
  77. case cacheStale, cacheMiss:
  78. if err := u.copyToLayer(name); err != nil {
  79. return err
  80. }
  81. err = u.base.Chtimes(name, atime, mtime)
  82. }
  83. if err != nil {
  84. return err
  85. }
  86. return u.layer.Chtimes(name, atime, mtime)
  87. }
  88. func (u *CacheOnReadFs) Chmod(name string, mode os.FileMode) error {
  89. st, _, err := u.cacheStatus(name)
  90. if err != nil {
  91. return err
  92. }
  93. switch st {
  94. case cacheLocal:
  95. case cacheHit:
  96. err = u.base.Chmod(name, mode)
  97. case cacheStale, cacheMiss:
  98. if err := u.copyToLayer(name); err != nil {
  99. return err
  100. }
  101. err = u.base.Chmod(name, mode)
  102. }
  103. if err != nil {
  104. return err
  105. }
  106. return u.layer.Chmod(name, mode)
  107. }
  108. func (u *CacheOnReadFs) Stat(name string) (os.FileInfo, error) {
  109. st, fi, err := u.cacheStatus(name)
  110. if err != nil {
  111. return nil, err
  112. }
  113. switch st {
  114. case cacheMiss:
  115. return u.base.Stat(name)
  116. default: // cacheStale has base, cacheHit and cacheLocal the layer os.FileInfo
  117. return fi, nil
  118. }
  119. }
  120. func (u *CacheOnReadFs) Rename(oldname, newname string) error {
  121. st, _, err := u.cacheStatus(oldname)
  122. if err != nil {
  123. return err
  124. }
  125. switch st {
  126. case cacheLocal:
  127. case cacheHit:
  128. err = u.base.Rename(oldname, newname)
  129. case cacheStale, cacheMiss:
  130. if err := u.copyToLayer(oldname); err != nil {
  131. return err
  132. }
  133. err = u.base.Rename(oldname, newname)
  134. }
  135. if err != nil {
  136. return err
  137. }
  138. return u.layer.Rename(oldname, newname)
  139. }
  140. func (u *CacheOnReadFs) Remove(name string) error {
  141. st, _, err := u.cacheStatus(name)
  142. if err != nil {
  143. return err
  144. }
  145. switch st {
  146. case cacheLocal:
  147. case cacheHit, cacheStale, cacheMiss:
  148. err = u.base.Remove(name)
  149. }
  150. if err != nil {
  151. return err
  152. }
  153. return u.layer.Remove(name)
  154. }
  155. func (u *CacheOnReadFs) RemoveAll(name string) error {
  156. st, _, err := u.cacheStatus(name)
  157. if err != nil {
  158. return err
  159. }
  160. switch st {
  161. case cacheLocal:
  162. case cacheHit, cacheStale, cacheMiss:
  163. err = u.base.RemoveAll(name)
  164. }
  165. if err != nil {
  166. return err
  167. }
  168. return u.layer.RemoveAll(name)
  169. }
  170. func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
  171. st, _, err := u.cacheStatus(name)
  172. if err != nil {
  173. return nil, err
  174. }
  175. switch st {
  176. case cacheLocal, cacheHit:
  177. default:
  178. if err := u.copyToLayer(name); err != nil {
  179. return nil, err
  180. }
  181. }
  182. if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 {
  183. bfi, err := u.base.OpenFile(name, flag, perm)
  184. if err != nil {
  185. return nil, err
  186. }
  187. lfi, err := u.layer.OpenFile(name, flag, perm)
  188. if err != nil {
  189. bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...?
  190. return nil, err
  191. }
  192. return &UnionFile{Base: bfi, Layer: lfi}, nil
  193. }
  194. return u.layer.OpenFile(name, flag, perm)
  195. }
  196. func (u *CacheOnReadFs) Open(name string) (File, error) {
  197. st, fi, err := u.cacheStatus(name)
  198. if err != nil {
  199. return nil, err
  200. }
  201. switch st {
  202. case cacheLocal:
  203. return u.layer.Open(name)
  204. case cacheMiss:
  205. bfi, err := u.base.Stat(name)
  206. if err != nil {
  207. return nil, err
  208. }
  209. if bfi.IsDir() {
  210. return u.base.Open(name)
  211. }
  212. if err := u.copyToLayer(name); err != nil {
  213. return nil, err
  214. }
  215. return u.layer.Open(name)
  216. case cacheStale:
  217. if !fi.IsDir() {
  218. if err := u.copyToLayer(name); err != nil {
  219. return nil, err
  220. }
  221. return u.layer.Open(name)
  222. }
  223. case cacheHit:
  224. if !fi.IsDir() {
  225. return u.layer.Open(name)
  226. }
  227. }
  228. // the dirs from cacheHit, cacheStale fall down here:
  229. bfile, _ := u.base.Open(name)
  230. lfile, err := u.layer.Open(name)
  231. if err != nil && bfile == nil {
  232. return nil, err
  233. }
  234. return &UnionFile{Base: bfile, Layer: lfile}, nil
  235. }
  236. func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error {
  237. err := u.base.Mkdir(name, perm)
  238. if err != nil {
  239. return err
  240. }
  241. return u.layer.MkdirAll(name, perm) // yes, MkdirAll... we cannot assume it exists in the cache
  242. }
  243. func (u *CacheOnReadFs) Name() string {
  244. return "CacheOnReadFs"
  245. }
  246. func (u *CacheOnReadFs) MkdirAll(name string, perm os.FileMode) error {
  247. err := u.base.MkdirAll(name, perm)
  248. if err != nil {
  249. return err
  250. }
  251. return u.layer.MkdirAll(name, perm)
  252. }
  253. func (u *CacheOnReadFs) Create(name string) (File, error) {
  254. bfh, err := u.base.Create(name)
  255. if err != nil {
  256. return nil, err
  257. }
  258. lfh, err := u.layer.Create(name)
  259. if err != nil {
  260. // oops, see comment about OS_TRUNC above, should we remove? then we have to
  261. // remember if the file did not exist before
  262. bfh.Close()
  263. return nil, err
  264. }
  265. return &UnionFile{Base: bfh, Layer: lfh}, nil
  266. }