Add basic cache purge by prefix

Author: bep

lib/cache.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/boltdb/bolt"
+
 	"github.com/asdine/storm"
 )
 
@@ -96,7 +98,7 @@ func (c *cacheHandler) handleRequest(rw http.ResponseWriter, req *http.Request)
 	}
 
 	if meta != nil {
-		c.logger.Debug("fileMeta", meta.Filename, "status", meta.StatusCode, "header", meta.Header)
+		c.logger.Debug("area", "cache", "filename", meta.Filename, "status", meta.StatusCode, "header", meta.Header)
 
 		f, err := c.getFile(relPath)
 		if err != nil {
@@ -201,7 +203,7 @@ func (c *cacheHandler) doWithDB(f func(db *storm.DB) error) error {
 }
 
 func (c *cacheHandler) openDB() (*storm.DB, error) {
-	return storm.Open(c.cfg.DBFilename)
+	return storm.Open(c.cfg.DBFilename, storm.BoltOptions(0600, &bolt.Options{Timeout: 5 * time.Second}))
 }
 
 // Plans:

lib/cache_maintenance.go

@@ -0,0 +1,58 @@
+// Copyright © 2017 Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lib
+
+import (
+	"github.com/asdine/storm"
+	"github.com/asdine/storm/q"
+)
+
+func (c *cacheHandler) purgePrefix(prefix string) error {
+	db, err := c.openDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	// TODO(bep) remove
+	var all []fileMeta
+	db.All(&all)
+	for _, a := range all {
+		c.logger.Debug("file", a.Filename)
+	}
+
+	// TODO(bep) a way to do this with an index.
+	q := db.Select(q.Re("Filename", "^"+prefix))
+
+	var files []fileMeta
+
+	if err := q.Find(&files); err != nil && err != storm.ErrNotFound {
+		return err
+	}
+
+	c.logger.Info("area", "cache", "tag", "purge", "prefix", prefix, "count", len(files))
+
+	for _, file := range files {
+		if err := db.DeleteStruct(&file); err != nil && err != storm.ErrNotFound {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *cacheHandler) shrinkTo(bytes int64) error {
+	return nil
+}

lib/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
+	"github.com/go-kit/kit/log/term"
 )
 
 type Config struct {
@@ -95,9 +96,32 @@ func LoadConfig(filename string) (Config, error) {
 
 func (c Config) CreateLogger() *Logger {
 	// TODO(bep) configure
-	logger := log.NewLogfmtLogger(os.Stdout)
-	logger = level.NewFilter(logger, level.AllowAll())
+	//logger := log.NewLogfmtLogger(os.Stdout)
+	colorFn := func(keyvals ...interface{}) term.FgBgColor {
+		for i := 0; i < len(keyvals)-1; i += 2 {
+			if keyvals[i] != "level" {
+				continue
+			}
+			switch keyvals[i+1] {
+			case "debug":
+				return term.FgBgColor{Fg: term.DarkGray}
+			case "info":
+				return term.FgBgColor{Fg: term.Gray}
+			case "warn":
+				return term.FgBgColor{Fg: term.Yellow}
+			case "error":
+				return term.FgBgColor{Fg: term.Red}
+			case "crit":
+				return term.FgBgColor{Fg: term.Gray, Bg: term.DarkRed}
+			default:
+				return term.FgBgColor{}
+			}
+		}
+		return term.FgBgColor{}
+	}
 
+	logger := term.NewLogger(os.Stdout, log.NewLogfmtLogger, colorFn)
+	logger = level.NewFilter(logger, level.AllowAll())
 	l := NewLogger(logger)
 
 	return l

lib/s3.go

@@ -103,8 +103,6 @@ func (s s3Client) getAndWrite(path string, host Host,
 		CreatedAt:  now,
 	}
 
-	s.logger.Debug("newFileMeta", fm.Filename, "status", fm.StatusCode, "header", fm.Header)
-
 	for k, v := range fm.Header {
 		for _, vv := range v {
 			rw.Header().Add(k, vv)
@@ -113,7 +111,7 @@ func (s s3Client) getAndWrite(path string, host Host,
 
 	rw.WriteHeader(resp.StatusCode)
 
-	s.logger.Debug("s3StreamFrom", path, "status", resp.StatusCode)
+	s.logger.Debug("area", "s3", "filename", fm.Filename, "status", resp.StatusCode)
 
 	var content io.Reader
 

lib/server.go

@@ -34,16 +34,26 @@ type Server struct {
 }
 
 type httpHandlers struct {
-	cfg    Config
-	logger *Logger
+	c *cacheHandler
 }
 
 func NewServer(cfg Config, logger *Logger) (*Server, error) {
 	// TODO(bep) validate config
 
 	h := http.NewServeMux()
-	mw := &httpHandlers{cfg: cfg, logger: logger}
+	c := newCacheHandler(cfg, logger)
+	mw := &httpHandlers{c: c}
+
+	var purger http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		prefix := r.FormValue("prefix")
+		if prefix != "" {
+			if err := c.purgePrefix(prefix); err != nil {
+				c.logger.Error("area", "cache", "tag", "purge", "prefix", prefix, "error", err)
+			}
+		}
+	}
 
+	h.Handle("/purge", mw.secure(purger))
 	h.Handle("/", mw.secure(mw.serveFile()))
 
 	tlsEnabled, err := cfg.isTLSConfigured()
@@ -87,12 +97,12 @@ func (s *Server) Shutdown(ctx context.Context) error {
 }
 
 func (m *httpHandlers) serveFile() http.HandlerFunc {
-	c := newCacheHandler(m.cfg, m.logger)
+
 	return func(w http.ResponseWriter, r *http.Request) {
 		// TODO containsDotDot https://github.com/golang/go/blob/f9cf8e5ab11c7ea3f1b9fde302c0a325df020b1a/src/net/http/fs.go#L665
-		err := c.handleRequest(w, r)
+		err := m.c.handleRequest(w, r)
 		if err != nil {
-			m.logger.Error("handleRequest", err)
+			m.c.logger.Error("handleRequest", err)
 			// TODO(bep) status code/err handling
 		}
 	}

lib/server_secure.go

@@ -23,7 +23,7 @@ import (
 func (m *httpHandlers) secure(h http.Handler) http.Handler {
 	// TODO(bep) => config
 	return secure.New(secure.Options{
-		AllowedHosts:         m.cfg.hostNames(),
+		AllowedHosts:         m.c.cfg.hostNames(),
 		HostsProxyHeaders:    []string{"X-Forwarded-Host"},
 		SSLRedirect:          true,
 		SSLHost:              "",