Upstream
/
pages-server
mirror of https://codeberg.org/Codeberg/pages-server.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move to []byte for caching and make it compile

This commit is contained in:
Moritz Marquardt 2024-03-26 07:38:15 +01:00
parent 5b6eecc75f
commit c4181d1206
11 changed files with 63 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/cache/interface.go vendored
View File

@ -4,7 +4,7 @@ import "time"
// ICache is an interface that defines how the pages server interacts with the cache.
type ICache interface {
Set(key string, value string, ttl time.Duration) error
Get(key string) (string, bool)
Set(key string, value []byte, ttl time.Duration) error
Get(key string) ([]byte, bool)
Remove(key string)
}

8
server/cache/memory.go vendored
View File

@ -9,16 +9,16 @@ type MCache struct {
mcache *mcache.CacheDriver
}
func (m *MCache) Set(key string, value string, ttl time.Duration) error {
func (m *MCache) Set(key string, value []byte, ttl time.Duration) error {
return m.mcache.Set(key, value, ttl)
}
func (m *MCache) Get(key string) (string, bool) {
func (m *MCache) Get(key string) ([]byte, bool) {
val, ok := m.mcache.Get(key)
if ok {
return val.(string), true
return val.([]byte), true
} else {
return "", false
return nil, false
}
}

11
server/cache/redis.go vendored
View File

@ -2,6 +2,7 @@ package cache
import (
"context"
"errors"
"github.com/redis/go-redis/v9"
"github.com/rs/zerolog/log"
"time"
@ -12,17 +13,17 @@ type RedisCache struct {
rdb *redis.Client
}
func (r *RedisCache) Set(key string, value string, ttl time.Duration) error {
func (r *RedisCache) Set(key string, value []byte, ttl time.Duration) error {
return r.rdb.Set(r.ctx, key, value, ttl).Err()
}
func (r *RedisCache) Get(key string) (string, bool) {
val, err := r.rdb.Get(r.ctx, key).Result()
func (r *RedisCache) Get(key string) ([]byte, bool) {
val, err := r.rdb.Get(r.ctx, key).Bytes()
if err != nil {
if err == redis.Nil {
if errors.Is(err, redis.Nil) {
log.Error().Err(err).Str("key", key).Msg("Couldn't request key from cache.")
}
return "", false
return nil, false
} else {
return val, true
}

8
server/certificates/cached_challengers.go
View File

@ -22,7 +22,7 @@ type AcmeTLSChallengeProvider struct {
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
return a.challengeCache.Set(domain, []byte(keyAuth), 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
@ -38,7 +38,7 @@ type AcmeHTTPChallengeProvider struct {
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
return a.challengeCache.Set(domain+"/"+token, []byte(keyAuth), 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
@ -60,12 +60,12 @@ func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) htt
// it's an acme request
if strings.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
if !ok || challenge == "" {
if !ok {
log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
ctx.String("no challenge for this token", http.StatusNotFound)
}
log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
ctx.String(challenge)
ctx.String(string(challenge))
return
}

2
server/certificates/certificates.go
View File

@ -57,7 +57,7 @@ func TLSConfig(mainDomainSuffix string,
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(domain, challenge)
cert, err := tlsalpn01.ChallengeCert(domain, string(challenge))
if err != nil {
return nil, err
}

28
server/gitea/cache.go
View File

@ -5,6 +5,8 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"github.com/rs/zerolog/log"
@ -41,6 +43,24 @@ type FileResponse struct {
Body []byte
}
func FileResponseFromMetadataString(metadataString string) FileResponse {
parts := strings.Split(metadataString, "\n")
res := FileResponse{
Exists: parts[0] == "true",
IsSymlink: parts[1] == "true",
ETag: parts[2],
MimeType: parts[3],
}
return res
}
func (f FileResponse) MetadataAsString() string {
return strconv.FormatBool(f.Exists) + "\n" +
strconv.FormatBool(f.IsSymlink) + "\n" +
f.ETag + "\n" +
f.MimeType + "\n"
}
func (f FileResponse) IsEmpty() bool {
return len(f.Body) == 0
}
@ -102,9 +122,13 @@ func (t *writeCacheReader) Close() error {
doWrite = false
}
if doWrite {
err := t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
err := t.cache.Set(t.cacheKey+"|Metadata", []byte(fc.MetadataAsString()), fileCacheTimeout)
if err != nil {
log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey)
log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Metadata")
}
err = t.cache.Set(t.cacheKey+"|Body", fc.Body, fileCacheTimeout)
if err != nil {
log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Body")
}
}
log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite)

27
server/gitea/client.go
View File

@ -116,12 +116,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
log.Trace().Msg("try file in cache")
// handle if cache entry exist
if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok {
cacheMetadataParts := strings.Split(cacheMetadata, "\n")
cache := FileResponse{
Exists: cacheMetadataParts[0] == "true",
IsSymlink: cacheMetadataParts[1] == "true",
ETag: cacheMetadataParts[2],
}
cache := FileResponseFromMetadataString(string(cacheMetadata))
cacheBodyString, _ := client.responseCache.Get(cacheKey + "|Body")
cache.Body = []byte(cacheBodyString)
// TODO: don't grab the content from the cache if the ETag matches?!
@ -174,12 +169,11 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
ETag: resp.Header.Get(ETagHeader),
}
log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body))
metadataStr := strconv.FormatBool(fileResponse.Exists) + "\n" + strconv.FormatBool(fileResponse.IsSymlink) + "\n" + fileResponse.ETag
if err := client.responseCache.Set(cacheKey+"|Metadata", metadataStr, fileCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey+"|Metadata", []byte(fileResponse.MetadataAsString()), fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
// TODO: Test with binary files, as we convert []byte to string! Using []byte values might makes more sense anyways.
if err := client.responseCache.Set(cacheKey+"|Body", string(fileResponse.Body), fileCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey+"|Body", fileResponse.Body, fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
@ -202,11 +196,10 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
ETag: resp.Header.Get(ETagHeader),
MimeType: mimeType,
}
// TODO: dafuq...
return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
case http.StatusNotFound:
if err := client.responseCache.Set(cacheKey+"|Metadata", "false\nfalse\n"+resp.Header.Get(ETagHeader), fileCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey+"|Metadata", []byte(FileResponse{ETag: resp.Header.Get(ETagHeader)}.MetadataAsString()), fileCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
@ -222,7 +215,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
if stamp, ok := client.responseCache.Get(cacheKey); ok {
if stamp == "" {
if len(stamp) == 0 {
log.Trace().Msgf("[cache] use branch %q not found", branchName)
return &BranchTimestamp{}, ErrorNotFound
}
@ -230,7 +223,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
// This comes from the refactoring of the caching library.
// The branch as reported by the API was stored in the cache, and I'm not sure if there are
// situations where it differs from the name in the request, hence this is left here.
stampParts := strings.SplitN(stamp, "", 2)
stampParts := strings.SplitN(string(stamp), "|", 2)
stampTime, _ := time.Parse(time.RFC3339, stampParts[0])
return &BranchTimestamp{
Branch: stampParts[1],
@ -242,7 +235,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound {
log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
if err := client.responseCache.Set(cacheKey, "", branchExistenceCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte{}, branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return &BranchTimestamp{}, ErrorNotFound
@ -259,7 +252,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
}
log.Trace().Msgf("set cache branch [%s] exist", branchName)
if err := client.responseCache.Set(cacheKey, stamp.Timestamp.Format(time.RFC3339)+"|"+stamp.Branch, branchExistenceCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte(stamp.Timestamp.Format(time.RFC3339)+"|"+stamp.Branch), branchExistenceCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return stamp, nil
@ -269,7 +262,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
if branch, ok := client.responseCache.Get(cacheKey); ok {
return branch, nil
return string(branch), nil
}
repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName)
@ -281,7 +274,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
}
branch := repo.DefaultBranch
if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
if err := client.responseCache.Set(cacheKey, []byte(branch), defaultBranchCacheTimeout); err != nil {
log.Error().Err(err).Msg("[cache] error on cache write")
}
return branch, nil

3
server/handler/handler.go
View File

@ -1,6 +1,7 @@
package handler
import (
"github.com/OrlovEvgeny/go-mcache"
"net/http"
"strings"
@ -23,7 +24,7 @@ const (
func Handler(
cfg config.ServerConfig,
giteaClient *gitea.Client,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
dnsLookupCache *mcache.CacheDriver, canonicalDomainCache, redirectsCache cache.ICache,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log.Debug().Msg("\n----------------------------------------------------------")

5
server/startup.go
View File

@ -5,6 +5,7 @@ import (
"crypto/tls"
"encoding/json"
"fmt"
"github.com/OrlovEvgeny/go-mcache"
"net"
"net/http"
"os"
@ -70,12 +71,12 @@ func Serve(ctx *cli.Context) error {
}
defer closeFn()
keyCache := cache.NewInMemoryCache()
keyCache := mcache.New()
challengeCache := cache.NewInMemoryCache()
// canonicalDomainCache stores canonical domains
canonicalDomainCache := cache.NewInMemoryCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewInMemoryCache()
dnsLookupCache := mcache.New()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewInMemoryCache()
// clientResponseCache stores responses from the Gitea server

6
server/upstream/domains.go
View File

@ -20,7 +20,7 @@ const canonicalDomainConfig = ".domains"
func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
// Check if this request is cached.
if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
domains := strings.Split(cachedValue, "\n")
domains := strings.Split(string(cachedValue), "\n")
for _, domain := range domains {
if domain == actualDomain {
valid = true
@ -33,7 +33,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
if err != nil && !errors.Is(err, gitea.ErrorNotFound) {
log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo)
// TODO: WTF we just continue?!
// TODO: WTF we just continue?! Seems fine as body is empty... :/
}
var domains []string
@ -63,7 +63,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain,
}
// Add result to cache.
_ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, strings.Join(domains, "\n"), canonicalDomainCacheTimeout)
_ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, []byte(strings.Join(domains, "\n")), canonicalDomainCacheTimeout)
// Return the first domain from the list and return if any of the domains
// matched the requested domain.

4
server/upstream/redirects.go
View File

@ -31,7 +31,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I
// Check for cached redirects
if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
redirects := []Redirect{}
err := json.Unmarshal([]byte(cachedValue), redirects)
err := json.Unmarshal(cachedValue, redirects)
if err != nil {
log.Error().Err(err).Msgf("could not parse redirects for key %s", cacheKey)
// It's okay to continue, the array stays empty.
@ -68,7 +68,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I
if err != nil {
log.Error().Err(err).Msgf("could not store redirects for key %s", cacheKey)
} else {
_ = redirectsCache.Set(cacheKey, string(redirectsJson), redirectsCacheTimeout)
_ = redirectsCache.Set(cacheKey, redirectsJson, redirectsCacheTimeout)
}
}
return redirects