From 33f7a5d0df0288b7625f77449134f8cda7078698 Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Sun, 22 Aug 2021 17:59:30 +0200 Subject: [PATCH] Commit all current changes before vacation... --- certificates.go | 74 ++++++++++++++++++++++++++++++++++++------------- go.mod | 1 + go.sum | 2 ++ handler.go | 11 ++++---- 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/certificates.go b/certificates.go index faccf7d..0c4ec18 100644 --- a/certificates.go +++ b/certificates.go @@ -25,6 +25,7 @@ import ( "time" "github.com/akrylysov/pogreb" + "github.com/reugn/equalizer" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/lego" @@ -107,12 +108,6 @@ var tlsConfig = &tls.Config{ } else { // request a new certificate - // TODO: rate-limit certificates per owner - // LE Rate Limits: - // - 300 new orders per account per 3 hours - // - 20 requests per second - // - 10 Accounts per IP per 3 hours - if bytes.Equal(sniBytes, MainDomainSuffix) { return nil, errors.New("won't request certificate for main domain, something really bad has happened") } @@ -123,6 +118,10 @@ var tlsConfig = &tls.Config{ return nil, err } key = x509.MarshalPKCS1PrivateKey(privateKey) + acmeClient, err := acmeClientFromPool(targetOwner) + if err != nil { + // TODO + } res, err := acmeClient.Certificate.Obtain(certificate.ObtainRequest{ Domains: []string{sni}, PrivateKey: key, @@ -259,7 +258,17 @@ func (u AcmeAccount) GetRegistration() *registration.Resource { func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey { return u.key } -var acmeClient *lego.Client + +// rate-limit certificates per owner, based on LE Rate Limits: +// - 300 new orders per account per 3 hours +// - 20 requests per second +// - 10 Accounts per IP per 3 hours +var acmeClientPool []*lego.Client +var lastAcmeClient = 0 +var acmeClientRequestLimit = equalizer.NewTokenBucket(10, time.Second) // LE allows 20 requests per second, but we want to give other applications a chancem so we want 10 here at most. +var acmeClientRegistrationLimit = equalizer.NewTokenBucket(5, time.Hour * 3) // LE allows 10 registrations in 3 hours per IP, we want at most 5 of them. +var acmeClientCertificateLimitPerRegistration = []*equalizer.TokenBucket{} +var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{} type AcmeTLSChallengeProvider struct{} var _ challenge.Provider = AcmeTLSChallengeProvider{} @@ -271,19 +280,31 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { return nil } -func init() { - FallbackCertificate() +func acmeClientFromPool(user string) (*lego.Client, error) { + userLimit, ok := acmeClientCertificateLimitPerUser[user] + if !ok { + // Each Codeberg user can only add 10 new domains per day. + userLimit = equalizer.NewTokenBucket(10, time.Hour * 24) + acmeClientCertificateLimitPerUser[user] = userLimit - var err error - keyDatabase, err = pogreb.Open("key-database.pogreb", &pogreb.Options{ - BackgroundSyncInterval: 30 * time.Second, - BackgroundCompactionInterval: 6 * time.Hour, - FileSystem: fs.OSMMap, - }) - if err != nil { - panic(err) + } + if !userLimit.Ask() { + return nil, errors.New("rate limit exceeded: 10 certificates per user per 24 hours") } + if len(acmeClientPool) < 1 { + acmeClientPool = append(acmeClientPool, newAcmeClient()) + acmeClientCertificateLimitPerRegistration = append(acmeClientCertificateLimitPerRegistration, equalizer.NewTokenBucket(290, time.Hour * 3)) + } + if !acmeClientCertificateLimitPerRegistration[(lastAcmeClient + 1) % len(acmeClientPool)].Ask() { + + } + equalizer.NewTokenBucket(290, time.Hour * 3) // LE allows 300 certificates per account, to be sure to catch it earlier, we limit that to 290. + + // TODO: limit domains by file in repo +} + +func newAcmeClient() *lego.Client { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { panic(err) @@ -295,7 +316,7 @@ func init() { config := lego.NewConfig(&myUser) config.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory") config.Certificate.KeyType = certcrypto.RSA2048 - acmeClient, err = lego.NewClient(config) + acmeClient, err := lego.NewClient(config) if err != nil { panic(err) } @@ -314,6 +335,21 @@ func init() { } else { log.Printf("Warning: not using ACME certificates as ACME_ACCEPT_TERMS is false!") } + return acmeClient +} + +func init() { + FallbackCertificate() + + var err error + keyDatabase, err = pogreb.Open("key-database.pogreb", &pogreb.Options{ + BackgroundSyncInterval: 30 * time.Second, + BackgroundCompactionInterval: 6 * time.Hour, + FileSystem: fs.OSMMap, + }) + if err != nil { + panic(err) + } // generate certificate for main domain if os.Getenv("ACME_ACCEPT_TERMS") != "true" || os.Getenv("DNS_PROVIDER") == "" { @@ -340,7 +376,7 @@ func init() { panic(err) } mainKey := x509.MarshalPKCS1PrivateKey(mainPrivateKey) - res, err := acmeClient.Certificate.Obtain(certificate.ObtainRequest{ + res, err := dnsAcmeClient.Certificate.Obtain(certificate.ObtainRequest{ Domains: []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, PrivateKey: mainKey, Bundle: true, diff --git a/go.mod b/go.mod index ba46849..5920606 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/andybalholm/brotli v1.0.3 // indirect github.com/go-acme/lego/v4 v4.4.0 github.com/klauspost/compress v1.13.1 // indirect + github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/valyala/fasthttp v1.28.0 github.com/valyala/fastjson v1.6.3 ) diff --git a/go.sum b/go.sum index 02603a8..0180b2f 100644 --- a/go.sum +++ b/go.sum @@ -409,6 +409,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjIi3L6QmL/V9vpJPbc/j/F2u55d3fs= +github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/handler.go b/handler.go index 909725f..ee52257 100644 --- a/handler.go +++ b/handler.go @@ -91,6 +91,7 @@ func handler(ctx *fasthttp.RequestCtx) { targetRepo = repo targetPath = strings.Trim(strings.Join(path, "/"), "/") targetBranch = branchTimestampResult.branch + targetOptions.BranchTimestamp = branchTimestampResult.timestamp if canonicalLink != "" { @@ -314,7 +315,7 @@ type fileResponse struct { } // getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch -// (or an empty time.Time if the branch doesn't exist) +// (or nil if the branch doesn't exist) func getBranchTimestamp(owner, repo, branch string) *branchTimestamp { if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok { if result == nil { @@ -394,7 +395,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t var res *fasthttp.Response var cachedResponse fileResponse var err error - if cachedValue, ok := fileResponseCache.Get(uri); ok { + if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok { cachedResponse = cachedValue.(fileResponse) } else { req = fasthttp.AcquireRequest() @@ -414,7 +415,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t optionsForIndexPages.AppendTrailingSlash = true for _, indexPage := range IndexPages { if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) { - _ = fileResponseCache.Set(uri, fileResponse{ + _ = fileResponseCache.Set(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) return true @@ -424,7 +425,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t ctx.Response.SetStatusCode(fasthttp.StatusNotFound) if res != nil { // Update cache if the request is fresh - _ = fileResponseCache.Set(uri, fileResponse{ + _ = fileResponseCache.Set(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{ exists: false, }, FileCacheTimeout) } @@ -484,7 +485,7 @@ func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, t cachedResponse.exists = true cachedResponse.mimeType = mimeType cachedResponse.body = cacheBodyWriter.Bytes() - _ = fileResponseCache.Set(uri, cachedResponse, FileCacheTimeout) + _ = fileResponseCache.Set(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout) } return true