From ac1b9520a5e25c8a2735b31acbf4597cd984c28e Mon Sep 17 00:00:00 2001 From: Max Stanley Date: Wed, 28 Feb 2024 23:14:01 +0000 Subject: [PATCH] Support canonical-domain-file configuration --- cli/flags.go | 6 ++ config/config.go | 19 ++--- config/setup.go | 3 + config/setup_test.go | 94 ++++++++++++++----------- server/certificates/certificates.go | 3 +- server/handler/handler.go | 3 + server/handler/handler_custom_domain.go | 5 +- server/handler/handler_raw_domain.go | 5 +- server/handler/handler_sub_domain.go | 11 +-- server/handler/try.go | 3 +- server/startup.go | 1 + server/upstream/domains.go | 10 +-- 12 files changed, 95 insertions(+), 68 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index 097cf4f..3998ba5 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -79,6 +79,12 @@ var ( Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", EnvVars: []string{"RAW_DOMAIN"}, }, + &cli.StringFlag{ + Name: "canonical-domain-file", + Usage: "specifies the file from which the canonical domain shall be specified in", + EnvVars: []string{"CANONICAL_DOMAIN_FILE"}, + Value: ".domains", + }, // ######################### // ### Page Server Setup ### diff --git a/config/config.go b/config/config.go index 6cb972b..7d7f8e9 100644 --- a/config/config.go +++ b/config/config.go @@ -9,15 +9,16 @@ type Config struct { } type ServerConfig struct { - Host string `default:"[::]"` - Port uint16 `default:"443"` - HttpPort uint16 `default:"80"` - HttpServerEnabled bool `default:"true"` - MainDomain string - RawDomain string - PagesBranches []string - AllowedCorsDomains []string - BlacklistedPaths []string + Host string `default:"[::]"` + Port uint16 `default:"443"` + HttpPort uint16 `default:"80"` + HttpServerEnabled bool `default:"true"` + MainDomain string + RawDomain string + CanonicalDomainFile string `default:".domains"` + PagesBranches []string + AllowedCorsDomains []string + BlacklistedPaths []string } type GiteaConfig struct { diff --git a/config/setup.go b/config/setup.go index e774084..e36c02d 100644 --- a/config/setup.go +++ b/config/setup.go @@ -75,6 +75,9 @@ func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { if ctx.IsSet("raw-domain") { config.RawDomain = ctx.String("raw-domain") } + if ctx.IsSet("canonical-domain-file") { + config.CanonicalDomainFile = ctx.String("canonical-domain-file") + } if ctx.IsSet("pages-branch") { config.PagesBranches = ctx.StringSlice("pages-branch") } diff --git a/config/setup_test.go b/config/setup_test.go index a863e2f..263c42b 100644 --- a/config/setup_test.go +++ b/config/setup_test.go @@ -136,15 +136,16 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T cfg := &Config{ LogLevel: "original", Server: ServerConfig{ - Host: "original", - Port: 8080, - HttpPort: 80, - HttpServerEnabled: false, - MainDomain: "original", - RawDomain: "original", - PagesBranches: []string{"original"}, - AllowedCorsDomains: []string{"original"}, - BlacklistedPaths: []string{"original"}, + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + CanonicalDomainFile: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, }, Gitea: GiteaConfig{ Root: "original", @@ -175,15 +176,16 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T expectedConfig := &Config{ LogLevel: "changed", Server: ServerConfig{ - Host: "changed", - Port: 8443, - HttpPort: 443, - HttpServerEnabled: true, - MainDomain: "changed", - RawDomain: "changed", - PagesBranches: []string{"changed"}, - AllowedCorsDomains: []string{"changed"}, - BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...), + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + CanonicalDomainFile: "changed", + PagesBranches: []string{"changed"}, + AllowedCorsDomains: []string{"changed"}, + BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...), }, Gitea: GiteaConfig{ Root: "changed", @@ -218,6 +220,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T // Server "--pages-domain", "changed", "--raw-domain", "changed", + "--canonical-domain-file", "changed", "--allowed-cors-domains", "changed", "--blacklisted-paths", "changed", "--pages-branch", "changed", @@ -270,27 +273,29 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes t, func(ctx *cli.Context) error { cfg := &ServerConfig{ - Host: "original", - Port: 8080, - HttpPort: 80, - HttpServerEnabled: false, - MainDomain: "original", - RawDomain: "original", - AllowedCorsDomains: []string{"original"}, - BlacklistedPaths: []string{"original"}, + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + CanonicalDomainFile: "original", + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, } mergeServerConfig(ctx, cfg) expectedConfig := &ServerConfig{ - Host: "changed", - Port: 8443, - HttpPort: 443, - HttpServerEnabled: true, - MainDomain: "changed", - RawDomain: "changed", - AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), - BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)), + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + CanonicalDomainFile: "changed", + AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), + BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)), } assert.Equal(t, expectedConfig, cfg) @@ -300,6 +305,7 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes []string{ "--pages-domain", "changed", "--raw-domain", "changed", + "--canonical-domain-file", "changed", "--allowed-cors-domains", "changed", "--blacklisted-paths", "changed", "--host", "changed", @@ -323,6 +329,7 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE {args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }}, {args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }}, {args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }}, + {args: []string{"--canonical-domain-file", "changed"}, callback: func(sc *ServerConfig) { sc.CanonicalDomainFile = "changed" }}, {args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }}, {args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }}, {args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }}, @@ -333,15 +340,16 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE t, func(ctx *cli.Context) error { cfg := ServerConfig{ - Host: "original", - Port: 8080, - HttpPort: 80, - HttpServerEnabled: false, - MainDomain: "original", - RawDomain: "original", - PagesBranches: []string{"original"}, - AllowedCorsDomains: []string{"original"}, - BlacklistedPaths: []string{"original"}, + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + CanonicalDomainFile: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, } expectedConfig := cfg diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index b638755..9977d52 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -31,6 +31,7 @@ func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, firstDefaultBranch string, + canonicalDomainConfig string, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, ) *tls.Config { @@ -79,7 +80,7 @@ func TLSConfig(mainDomainSuffix string, TargetRepo: targetRepo, TargetBranch: targetBranch, } - _, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache) + _, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainConfig, canonicalDomainCache) if !valid { // We shouldn't obtain a certificate when we cannot check if the // repository has specified this domain in the `.domains` file. diff --git a/server/handler/handler.go b/server/handler/handler.go index ffc3400..5750cf8 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -92,6 +92,7 @@ func Handler( cfg.MainDomain, trimmedHost, pathElements, + cfg.CanonicalDomainFile, canonicalDomainCache, redirectsCache) } else if strings.HasSuffix(trimmedHost, cfg.MainDomain) { log.Debug().Msg("subdomain request detected") @@ -100,6 +101,7 @@ func Handler( cfg.PagesBranches, trimmedHost, pathElements, + cfg.CanonicalDomainFile, canonicalDomainCache, redirectsCache) } else { log.Debug().Msg("custom domain request detected") @@ -108,6 +110,7 @@ func Handler( trimmedHost, pathElements, cfg.PagesBranches[0], + cfg.CanonicalDomainFile, dnsLookupCache, canonicalDomainCache, redirectsCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 82953f9..a0ffe6e 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -19,6 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g trimmedHost string, pathElements []string, firstDefaultBranch string, + canonicalDomainConfig string, dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from custom domains @@ -47,7 +48,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g TargetBranch: targetBranch, TargetPath: path.Join(pathParts...), }, canonicalLink); works { - canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache) + canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainConfig, canonicalDomainCache) if !valid { html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest) return @@ -64,7 +65,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g } log.Debug().Msg("tryBranch, now trying upstream 7") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) return } diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index f48e8e4..1aba584 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -19,6 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie mainDomainSuffix string, trimmedHost string, pathElements []string, + canonicalDomainConfig string, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve raw content from RawDomain @@ -45,7 +46,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie TargetPath: path.Join(pathElements[3:]...), }, true); works { log.Trace().Msg("tryUpstream: serve raw domain with specified branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) return } log.Debug().Msg("missing branch info") @@ -62,7 +63,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie TargetPath: path.Join(pathElements[2:]...), }, true); works { log.Trace().Msg("tryUpstream: serve raw domain with default branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage(ctx, fmt.Sprintf("raw domain could not find repo %s/%s or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo), diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 806fe7f..9a19e81 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -21,6 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite defaultPagesBranches []string, trimmedHost string, pathElements []string, + canonicalDomainConfig string, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from subdomains of MainDomainSuffix @@ -53,7 +54,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[2:]...), }, true); works { log.Trace().Msg("tryUpstream: serve with specified repo and branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage( ctx, @@ -85,7 +86,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[1:]...), }, true); works { log.Trace().Msg("tryUpstream: serve default pages repo with specified branch") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) } else { html.ReturnErrorPage( ctx, @@ -110,7 +111,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements[1:]...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 5") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) return } } @@ -126,7 +127,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) return } } @@ -141,7 +142,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite TargetPath: path.Join(pathElements...), }, false); works { log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache) + tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainConfig, canonicalDomainCache, redirectsCache) return } diff --git a/server/handler/try.go b/server/handler/try.go index 145b1a9..7f9f631 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -17,12 +17,13 @@ import ( func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost string, options *upstream.Options, + canonicalDomainConfig string, canonicalDomainCache cache.ICache, redirectsCache cache.ICache, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { - canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache) + canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainConfig, canonicalDomainCache) if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) { canonicalPath := ctx.Req.RequestURI if options.TargetRepo != defaultPagesRepo { diff --git a/server/startup.go b/server/startup.go index ffdabb7..bd242d3 100644 --- a/server/startup.go +++ b/server/startup.go @@ -108,6 +108,7 @@ func Serve(ctx *cli.Context) error { giteaClient, acmeClient, cfg.Server.PagesBranches[0], + cfg.Server.CanonicalDomainFile, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB, )) diff --git a/server/upstream/domains.go b/server/upstream/domains.go index d53d586..b5d93fc 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -14,12 +14,12 @@ import ( // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. var canonicalDomainCacheTimeout = 15 * time.Minute -const canonicalDomainConfig = ".domains" - // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) { +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix, canonicalDomainConfig string, canonicalDomainCache cache.ICache) (domain string, valid bool) { + canonicalDomainCacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch + "/" + canonicalDomainConfig + // Check if this request is cached. - if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { + if cachedValue, ok := canonicalDomainCache.Get(canonicalDomainCacheKey); ok { domains := cachedValue.([]string) for _, domain := range domains { if domain == actualDomain { @@ -62,7 +62,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, } // Add result to cache. - _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout) + _ = canonicalDomainCache.Set(canonicalDomainCacheKey, domains, canonicalDomainCacheTimeout) // Return the first domain from the list and return if any of the domains // matched the requested domain.