From c4181d1206362abf32d39ecd3078494e56c99dea Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Tue, 26 Mar 2024 07:38:15 +0100 Subject: [PATCH] Move to []byte for caching and make it compile --- server/cache/interface.go | 4 ++-- server/cache/memory.go | 8 +++---- server/cache/redis.go | 11 +++++---- server/certificates/cached_challengers.go | 8 +++---- server/certificates/certificates.go | 2 +- server/gitea/cache.go | 28 +++++++++++++++++++++-- server/gitea/client.go | 27 ++++++++-------------- server/handler/handler.go | 3 ++- server/startup.go | 5 ++-- server/upstream/domains.go | 6 ++--- server/upstream/redirects.go | 4 ++-- 11 files changed, 63 insertions(+), 43 deletions(-) diff --git a/server/cache/interface.go b/server/cache/interface.go index acc4b5b..bdf487a 100644 --- a/server/cache/interface.go +++ b/server/cache/interface.go @@ -4,7 +4,7 @@ import "time" // ICache is an interface that defines how the pages server interacts with the cache. type ICache interface { - Set(key string, value string, ttl time.Duration) error - Get(key string) (string, bool) + Set(key string, value []byte, ttl time.Duration) error + Get(key string) ([]byte, bool) Remove(key string) } diff --git a/server/cache/memory.go b/server/cache/memory.go index 7dfc18c..30ebcfe 100644 --- a/server/cache/memory.go +++ b/server/cache/memory.go @@ -9,16 +9,16 @@ type MCache struct { mcache *mcache.CacheDriver } -func (m *MCache) Set(key string, value string, ttl time.Duration) error { +func (m *MCache) Set(key string, value []byte, ttl time.Duration) error { return m.mcache.Set(key, value, ttl) } -func (m *MCache) Get(key string) (string, bool) { +func (m *MCache) Get(key string) ([]byte, bool) { val, ok := m.mcache.Get(key) if ok { - return val.(string), true + return val.([]byte), true } else { - return "", false + return nil, false } } diff --git a/server/cache/redis.go b/server/cache/redis.go index d2ba8df..3263d8d 100644 --- a/server/cache/redis.go +++ b/server/cache/redis.go @@ -2,6 +2,7 @@ package cache import ( "context" + "errors" "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "time" @@ -12,17 +13,17 @@ type RedisCache struct { rdb *redis.Client } -func (r *RedisCache) Set(key string, value string, ttl time.Duration) error { +func (r *RedisCache) Set(key string, value []byte, ttl time.Duration) error { return r.rdb.Set(r.ctx, key, value, ttl).Err() } -func (r *RedisCache) Get(key string) (string, bool) { - val, err := r.rdb.Get(r.ctx, key).Result() +func (r *RedisCache) Get(key string) ([]byte, bool) { + val, err := r.rdb.Get(r.ctx, key).Bytes() if err != nil { - if err == redis.Nil { + if errors.Is(err, redis.Nil) { log.Error().Err(err).Str("key", key).Msg("Couldn't request key from cache.") } - return "", false + return nil, false } else { return val, true } diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index d11d3f4..eb518f2 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -22,7 +22,7 @@ type AcmeTLSChallengeProvider struct { var _ challenge.Provider = AcmeTLSChallengeProvider{} func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error { - return a.challengeCache.Set(domain, keyAuth, 1*time.Hour) + return a.challengeCache.Set(domain, []byte(keyAuth), 1*time.Hour) } func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { @@ -38,7 +38,7 @@ type AcmeHTTPChallengeProvider struct { var _ challenge.Provider = AcmeHTTPChallengeProvider{} func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error { - return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour) + return a.challengeCache.Set(domain+"/"+token, []byte(keyAuth), 1*time.Hour) } func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { @@ -60,12 +60,12 @@ func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) htt // it's an acme request if strings.HasPrefix(ctx.Path(), challengePath) { challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath)) - if !ok || challenge == "" { + if !ok { log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain) ctx.String("no challenge for this token", http.StatusNotFound) } log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain) - ctx.String(challenge) + ctx.String(string(challenge)) return } diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index db1ea96..5ff8b08 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -57,7 +57,7 @@ func TLSConfig(mainDomainSuffix string, if !ok { return nil, errors.New("no challenge for this domain") } - cert, err := tlsalpn01.ChallengeCert(domain, challenge) + cert, err := tlsalpn01.ChallengeCert(domain, string(challenge)) if err != nil { return nil, err } diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 97024a1..23995d1 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net/http" + "strconv" + "strings" "time" "github.com/rs/zerolog/log" @@ -41,6 +43,24 @@ type FileResponse struct { Body []byte } +func FileResponseFromMetadataString(metadataString string) FileResponse { + parts := strings.Split(metadataString, "\n") + res := FileResponse{ + Exists: parts[0] == "true", + IsSymlink: parts[1] == "true", + ETag: parts[2], + MimeType: parts[3], + } + return res +} + +func (f FileResponse) MetadataAsString() string { + return strconv.FormatBool(f.Exists) + "\n" + + strconv.FormatBool(f.IsSymlink) + "\n" + + f.ETag + "\n" + + f.MimeType + "\n" +} + func (f FileResponse) IsEmpty() bool { return len(f.Body) == 0 } @@ -102,9 +122,13 @@ func (t *writeCacheReader) Close() error { doWrite = false } if doWrite { - err := t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + err := t.cache.Set(t.cacheKey+"|Metadata", []byte(fc.MetadataAsString()), fileCacheTimeout) if err != nil { - log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey) + log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Metadata") + } + err = t.cache.Set(t.cacheKey+"|Body", fc.Body, fileCacheTimeout) + if err != nil { + log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey+"|Body") } } log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite) diff --git a/server/gitea/client.go b/server/gitea/client.go index 067bad9..8287d31 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -116,12 +116,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str log.Trace().Msg("try file in cache") // handle if cache entry exist if cacheMetadata, ok := client.responseCache.Get(cacheKey + "|Metadata"); ok { - cacheMetadataParts := strings.Split(cacheMetadata, "\n") - cache := FileResponse{ - Exists: cacheMetadataParts[0] == "true", - IsSymlink: cacheMetadataParts[1] == "true", - ETag: cacheMetadataParts[2], - } + cache := FileResponseFromMetadataString(string(cacheMetadata)) cacheBodyString, _ := client.responseCache.Get(cacheKey + "|Body") cache.Body = []byte(cacheBodyString) // TODO: don't grab the content from the cache if the ETag matches?! @@ -174,12 +169,11 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str ETag: resp.Header.Get(ETagHeader), } log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body)) - metadataStr := strconv.FormatBool(fileResponse.Exists) + "\n" + strconv.FormatBool(fileResponse.IsSymlink) + "\n" + fileResponse.ETag - if err := client.responseCache.Set(cacheKey+"|Metadata", metadataStr, fileCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey+"|Metadata", []byte(fileResponse.MetadataAsString()), fileCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } // TODO: Test with binary files, as we convert []byte to string! Using []byte values might makes more sense anyways. - if err := client.responseCache.Set(cacheKey+"|Body", string(fileResponse.Body), fileCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey+"|Body", fileResponse.Body, fileCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } @@ -202,11 +196,10 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str ETag: resp.Header.Get(ETagHeader), MimeType: mimeType, } - // TODO: dafuq... return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil case http.StatusNotFound: - if err := client.responseCache.Set(cacheKey+"|Metadata", "false\nfalse\n"+resp.Header.Get(ETagHeader), fileCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey+"|Metadata", []byte(FileResponse{ETag: resp.Header.Get(ETagHeader)}.MetadataAsString()), fileCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } @@ -222,7 +215,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) if stamp, ok := client.responseCache.Get(cacheKey); ok { - if stamp == "" { + if len(stamp) == 0 { log.Trace().Msgf("[cache] use branch %q not found", branchName) return &BranchTimestamp{}, ErrorNotFound } @@ -230,7 +223,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam // This comes from the refactoring of the caching library. // The branch as reported by the API was stored in the cache, and I'm not sure if there are // situations where it differs from the name in the request, hence this is left here. - stampParts := strings.SplitN(stamp, "", 2) + stampParts := strings.SplitN(string(stamp), "|", 2) stampTime, _ := time.Parse(time.RFC3339, stampParts[0]) return &BranchTimestamp{ Branch: stampParts[1], @@ -242,7 +235,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Trace().Msgf("[cache] set cache branch %q not found", branchName) - if err := client.responseCache.Set(cacheKey, "", branchExistenceCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte{}, branchExistenceCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return &BranchTimestamp{}, ErrorNotFound @@ -259,7 +252,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam } log.Trace().Msgf("set cache branch [%s] exist", branchName) - if err := client.responseCache.Set(cacheKey, stamp.Timestamp.Format(time.RFC3339)+"|"+stamp.Branch, branchExistenceCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte(stamp.Timestamp.Format(time.RFC3339)+"|"+stamp.Branch), branchExistenceCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return stamp, nil @@ -269,7 +262,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) if branch, ok := client.responseCache.Get(cacheKey); ok { - return branch, nil + return string(branch), nil } repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName) @@ -281,7 +274,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str } branch := repo.DefaultBranch - if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { + if err := client.responseCache.Set(cacheKey, []byte(branch), defaultBranchCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } return branch, nil diff --git a/server/handler/handler.go b/server/handler/handler.go index ffc3400..c68ee06 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -1,6 +1,7 @@ package handler import ( + "github.com/OrlovEvgeny/go-mcache" "net/http" "strings" @@ -23,7 +24,7 @@ const ( func Handler( cfg config.ServerConfig, giteaClient *gitea.Client, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, + dnsLookupCache *mcache.CacheDriver, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { log.Debug().Msg("\n----------------------------------------------------------") diff --git a/server/startup.go b/server/startup.go index 149a07d..a639e37 100644 --- a/server/startup.go +++ b/server/startup.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "github.com/OrlovEvgeny/go-mcache" "net" "net/http" "os" @@ -70,12 +71,12 @@ func Serve(ctx *cli.Context) error { } defer closeFn() - keyCache := cache.NewInMemoryCache() + keyCache := mcache.New() challengeCache := cache.NewInMemoryCache() // canonicalDomainCache stores canonical domains canonicalDomainCache := cache.NewInMemoryCache() // dnsLookupCache stores DNS lookups for custom domains - dnsLookupCache := cache.NewInMemoryCache() + dnsLookupCache := mcache.New() // redirectsCache stores redirects in _redirects files redirectsCache := cache.NewInMemoryCache() // clientResponseCache stores responses from the Gitea server diff --git a/server/upstream/domains.go b/server/upstream/domains.go index ac070bf..b827966 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -20,7 +20,7 @@ const canonicalDomainConfig = ".domains" func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) { // Check if this request is cached. if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { - domains := strings.Split(cachedValue, "\n") + domains := strings.Split(string(cachedValue), "\n") for _, domain := range domains { if domain == actualDomain { valid = true @@ -33,7 +33,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig) if err != nil && !errors.Is(err, gitea.ErrorNotFound) { log.Error().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo) - // TODO: WTF we just continue?! + // TODO: WTF we just continue?! Seems fine as body is empty... :/ } var domains []string @@ -63,7 +63,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, } // Add result to cache. - _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, strings.Join(domains, "\n"), canonicalDomainCacheTimeout) + _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, []byte(strings.Join(domains, "\n")), canonicalDomainCacheTimeout) // Return the first domain from the list and return if any of the domains // matched the requested domain. diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index 3a19928..d09214b 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -31,7 +31,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I // Check for cached redirects if cachedValue, ok := redirectsCache.Get(cacheKey); ok { redirects := []Redirect{} - err := json.Unmarshal([]byte(cachedValue), redirects) + err := json.Unmarshal(cachedValue, redirects) if err != nil { log.Error().Err(err).Msgf("could not parse redirects for key %s", cacheKey) // It's okay to continue, the array stays empty. @@ -68,7 +68,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I if err != nil { log.Error().Err(err).Msgf("could not store redirects for key %s", cacheKey) } else { - _ = redirectsCache.Set(cacheKey, string(redirectsJson), redirectsCacheTimeout) + _ = redirectsCache.Set(cacheKey, redirectsJson, redirectsCacheTimeout) } } return redirects