revert changes

DNS caching is also limited to 30 seconds now instead of 5 minutes

.woodpecker/build.yml

@@ -11,7 +11,7 @@ depends_on:
 steps:
   # use vendor to cache dependencies
   vendor:
-    image: golang:1.21
+    image: golang:1.22
     commands:
       - go mod vendor
 

Justfile

@@ -1,13 +1,13 @@
 CGO_FLAGS := '-extldflags "-static" -linkmode external'
 TAGS      := 'sqlite sqlite_unlock_notify netgo'
 
-dev:
+dev *FLAGS:
     #!/usr/bin/env bash
     set -euxo pipefail
     set -a # automatically export all variables
     source .env-dev
     set +a
-    go run -tags '{{TAGS}}' .
+    go run -tags '{{TAGS}}' . {{FLAGS}}
 
 build:
     CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./

README.md

@@ -124,3 +124,18 @@ now these pages should work:
 - <https://momar.localhost.mock.directory:4430/ci-testing/>
 - <https://momar.localhost.mock.directory:4430/pag/@master/>
 - <https://mock-pages.codeberg-test.org:4430/README.md>
+
+### Profiling
+
+> This section is just a collection of commands for quick reference. If you want to learn more about profiling read [this](https://go.dev/doc/diagnostics) article or google `golang profiling`.
+
+First enable profiling by supplying the cli arg `--enable-profiling` or using the environment variable `EENABLE_PROFILING`.
+
+Get cpu and mem stats:
+
+```bash
+go tool pprof -raw -output=cpu.txt 'http://localhost:9999/debug/pprof/profile?seconds=60' &
+curl -so mem.txt 'http://localhost:9999/debug/pprof/heap?seconds=60'
+```
+
+More endpoints are documented here: <https://pkg.go.dev/net/http/pprof>

cli/flags.go

@@ -139,6 +139,18 @@ var (
 			EnvVars: []string{"CONFIG_FILE"},
 		},
 
+		&cli.BoolFlag{
+			Name:    "enable-profiling",
+			Usage:   "enables the go http profiling endpoints",
+			EnvVars: []string{"ENABLE_PROFILING"},
+		},
+		&cli.StringFlag{
+			Name:    "profiling-address",
+			Usage:   "specify ip address and port the profiling server should listen on",
+			EnvVars: []string{"PROFILING_ADDRESS"},
+			Value:   "localhost:9999",
+		},
+
 		// ############################
 		// ### ACME Client Settings ###
 		// ############################

flake.lock

@@ -19,11 +19,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1714030708,
-        "narHash": "sha256-JOGPOxa8N6ySzB7SQBsh0OVz+UXZriyahgvfNHMIY0Y=",
+        "lastModified": 1714314149,
+        "narHash": "sha256-yNAevSKF4krRWacmLUsLK7D7PlfuY3zF0lYnGYNi9vQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b0d52b31f7f4d80f8bf38f0253652125579c35ff",
+        "rev": "cf8cc1201be8bc71b7cbbbdaf349b22f4f99c7ae",
         "type": "github"
       },
       "original": {

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/creasty/defaults v1.7.0
 	github.com/go-acme/lego/v4 v4.5.3
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/joho/godotenv v1.4.0
 	github.com/lib/pq v1.10.7
 	github.com/mattn/go-sqlite3 v1.14.16

go.sum

@@ -323,6 +323,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=

server/certificates/certificates.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-acme/lego/v4/certificate"
 	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
 	"github.com/go-acme/lego/v4/lego"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/reugn/equalizer"
 	"github.com/rs/zerolog/log"
 
@@ -31,11 +32,13 @@ func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
 	acmeClient *AcmeClient,
 	firstDefaultBranch string,
-	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache,
+	challengeCache, canonicalDomainCache cache.ICache,
 	certDB database.CertDB,
 	noDNS01 bool,
 	rawDomain string,
 ) *tls.Config {
+	keyCache := expirable.NewLRU[string, *tls.Certificate](32, nil, 24*time.Hour)
+
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@@ -86,7 +89,7 @@ func TLSConfig(mainDomainSuffix string,
 				}
 			} else {
 				var targetRepo, targetBranch string
-				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch)
 				if targetOwner == "" {
 					// DNS not set up, return main certificate to redirect to the docs
 					domain = mainDomainSuffix
@@ -107,7 +110,7 @@ func TLSConfig(mainDomainSuffix string,
 
 			if tlsCertificate, ok := keyCache.Get(domain); ok {
 				// we can use an existing certificate object
-				return tlsCertificate.(*tls.Certificate), nil
+				return tlsCertificate, nil
 			}
 
 			var tlsCertificate *tls.Certificate
@@ -132,9 +135,8 @@ func TLSConfig(mainDomainSuffix string,
 				}
 			}
 
-			if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil {
-				return nil, err
-			}
+			keyCache.Add(domain, tlsCertificate)
+
 			return tlsCertificate, nil
 		},
 		NextProtos: []string{
@@ -186,11 +188,10 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
 
 	// TODO: document & put into own function
 	if !strings.EqualFold(sni, mainDomainSuffix) {
-		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
+		tlsCertificate.Leaf, err = leaf(&tlsCertificate)
 		if err != nil {
-			return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err)
+			return nil, err
 		}
-
 		// renew certificates 7 days before they expire
 		if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) {
 			// TODO: use ValidTill of custom cert struct
@@ -291,6 +292,7 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew
 			}
 			leaf, err := leaf(&tlsCertificate)
 			if err == nil && leaf.NotAfter.After(time.Now()) {
+				tlsCertificate.Leaf = leaf
 				// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
 				renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
 				if err := keyDatabase.Put(name, renew); err != nil {
@@ -388,11 +390,20 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *Acm
 	}
 }
 
-// leaf returns the parsed leaf certificate, either from c.leaf or by parsing
+// leaf returns the parsed leaf certificate, either from c.Leaf or by parsing
 // the corresponding c.Certificate[0].
+// After successfully parsing the cert c.Leaf gets set to the parsed cert.
 func leaf(c *tls.Certificate) (*x509.Certificate, error) {
 	if c.Leaf != nil {
 		return c.Leaf, nil
 	}
-	return x509.ParseCertificate(c.Certificate[0])
+
+	leaf, err := x509.ParseCertificate(c.Certificate[0])
+	if err != nil {
+		return nil, fmt.Errorf("tlsCert - failed to parse leaf: %w", err)
+	}
+
+	c.Leaf = leaf
+
+	return leaf, err
 }

server/dns/dns.go

@@ -5,22 +5,26 @@ import (
 	"strings"
 	"time"
 
-	"codeberg.org/codeberg/pages/server/cache"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var lookupCacheTimeout = 15 * time.Minute
+const (
+	lookupCacheValidity = 30 * time.Second
+	defaultPagesRepo    = "pages"
+)
 
-var defaultPagesRepo = "pages"
+// TODO(#316): refactor to not use global variables
+var lookupCache *expirable.LRU[string, string] = expirable.NewLRU[string, string](4096, nil, lookupCacheValidity)
 
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string) (targetOwner, targetRepo, targetBranch string) {
 	// Get CNAME or TXT
 	var cname string
 	var err error
-	if cachedName, ok := dnsLookupCache.Get(domain); ok {
-		cname = cachedName.(string)
+
+	if entry, ok := lookupCache.Get(domain); ok {
+		cname = entry
 	} else {
 		cname, err = net.LookupCNAME(domain)
 		cname = strings.TrimSuffix(cname, ".")
@@ -38,7 +42,7 @@ func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLo
 				}
 			}
 		}
-		_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
+		_ = lookupCache.Add(domain, cname)
 	}
 	if cname == "" {
 		return

server/handler/handler.go

@@ -23,7 +23,7 @@ const (
 func Handler(
 	cfg config.ServerConfig,
 	giteaClient *gitea.Client,
-	dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		log.Debug().Msg("\n----------------------------------------------------------")
@@ -108,7 +108,7 @@ func Handler(
 				trimmedHost,
 				pathElements,
 				cfg.PagesBranches[0],
-				dnsLookupCache, canonicalDomainCache, redirectsCache)
+				canonicalDomainCache, redirectsCache)
 		}
 	}
 }

server/handler/handler_custom_domain.go

@@ -19,10 +19,10 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 	trimmedHost string,
 	pathElements []string,
 	firstDefaultBranch string,
-	dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
+	canonicalDomainCache, redirectsCache cache.ICache,
 ) {
 	// Serve pages from custom domains
-	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
+	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch)
 	if targetOwner == "" {
 		html.ReturnErrorPage(ctx,
 			"could not obtain repo owner from custom domain",
@@ -53,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 			return
 		} else if canonicalDomain != trimmedHost {
 			// only redirect if the target is also a codeberg page!
-			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
+			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch)
 			if targetOwner != "" {
 				ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
 				return

server/handler/handler_test.go

@@ -29,7 +29,7 @@ func TestHandlerPerformance(t *testing.T) {
 		AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
 		PagesBranches:      []string{"pages"},
 	}
-	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache())
+	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
 
 	testCase := func(uri string, status int) {
 		t.Run(uri, func(t *testing.T) {

server/profiling.go

@@ -0,0 +1,21 @@
+package server
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/rs/zerolog/log"
+)
+
+func StartProfilingServer(listeningAddress string) {
+	server := &http.Server{
+		Addr:    listeningAddress,
+		Handler: http.DefaultServeMux,
+	}
+
+	log.Info().Msgf("Starting debug server on %s", listeningAddress)
+
+	go func() {
+		log.Fatal().Err(server.ListenAndServe()).Msg("Failed to start debug server")
+	}()
+}

server/startup.go

@@ -3,7 +3,6 @@ package server
 import (
 	"context"
 	"crypto/tls"
-	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -43,9 +42,6 @@ func Serve(ctx *cli.Context) error {
 	}
 	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
 
-	foo, _ := json.Marshal(cfg)
-	log.Trace().RawJSON("config", foo).Msg("starting server with config")
-
 	listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)
 	listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort)
 
@@ -70,12 +66,9 @@ func Serve(ctx *cli.Context) error {
 	}
 	defer closeFn()
 
-	keyCache := cache.NewInMemoryCache()
 	challengeCache := cache.NewInMemoryCache()
 	// canonicalDomainCache stores canonical domains
 	canonicalDomainCache := cache.NewInMemoryCache()
-	// dnsLookupCache stores DNS lookups for custom domains
-	dnsLookupCache := cache.NewInMemoryCache()
 	// redirectsCache stores redirects in _redirects files
 	redirectsCache := cache.NewInMemoryCache()
 	// clientResponseCache stores responses from the Gitea server
@@ -108,7 +101,7 @@ func Serve(ctx *cli.Context) error {
 		giteaClient,
 		acmeClient,
 		cfg.Server.PagesBranches[0],
-		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
+		challengeCache, canonicalDomainCache,
 		certDB,
 		cfg.ACME.NoDNS01,
 		cfg.Server.RawDomain,
@@ -133,8 +126,12 @@ func Serve(ctx *cli.Context) error {
 		}()
 	}
 
+	if ctx.IsSet("enable-profiling") {
+		StartProfilingServer(ctx.String("profiling-address"))
+	}
+
 	// Create ssl handler based on settings
-	sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache)
+	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
 
 	// Start the ssl listener
 	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())