From c1fbe861fe793eb335cab63a524e5ae2bb3fb7f1 Mon Sep 17 00:00:00 2001 From: crapStone Date: Sun, 11 Feb 2024 12:43:25 +0000 Subject: [PATCH 1/8] rename gitea to forge in html error messages (#287) closes #286 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/287 Reviewed-by: Andreas Shimokawa Co-authored-by: crapStone Co-committed-by: crapStone --- server/certificates/certificates.go | 2 +- server/handler/try.go | 2 +- server/upstream/domains.go | 2 +- server/upstream/upstream.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 3ae891a..4fb99fa 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -199,7 +199,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' { + if useDnsProvider && domains[0] != "" && domains[0][0] == '*' { domains = domains[1:] } diff --git a/server/handler/try.go b/server/handler/try.go index 838ae27..bb79dce 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -41,7 +41,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "gitea client failed", ctx.StatusCode) + html.ReturnErrorPage(ctx, "forge client failed", ctx.StatusCode) } } diff --git a/server/upstream/domains.go b/server/upstream/domains.go index eb30394..230f268 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -41,7 +41,7 @@ func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, domain = strings.TrimSpace(domain) domain = strings.TrimPrefix(domain, "http://") domain = strings.TrimPrefix(domain, "https://") - if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { + if domain != "" && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') { domains = append(domains, domain) } if domain == actualDomain { diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 1a444e4..4a85503 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -57,7 +57,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { - html.ReturnErrorPage(ctx, "gitea client: either repo owner or name info is missing", http.StatusBadRequest) + html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest) return true } @@ -153,16 +153,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi var msg string if err != nil { - msg = "gitea client: returned unexpected error" + msg = "forge client: returned unexpected error" log.Error().Err(err).Msg(msg) msg = fmt.Sprintf("%s: '%v'", msg, err) } if reader == nil { - msg = "gitea client: returned no reader" + msg = "forge client: returned no reader" log.Error().Msg(msg) } if statusCode != http.StatusOK { - msg = fmt.Sprintf("gitea client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) + msg = fmt.Sprintf("forge client: couldn't fetch contents: %d - %s", statusCode, http.StatusText(statusCode)) log.Error().Msg(msg) } From 7e80ade24b8aac072804122b343a2a1a70667983 Mon Sep 17 00:00:00 2001 From: crapStone Date: Thu, 15 Feb 2024 16:08:29 +0000 Subject: [PATCH 2/8] Add config file and rework cli parsing and passing of config values (#263) Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/263 Reviewed-by: 6543 <6543@obermui.de> Co-authored-by: crapStone Co-committed-by: crapStone --- .env-dev | 11 + .vscode/launch.json | 26 + Justfile | 15 +- {cmd => cli}/certs.go | 6 +- {cmd => cli}/flags.go | 52 +- cli/setup.go | 39 ++ cmd/main.go | 150 ------ cmd/setup.go | 64 --- config/assets/test_config.toml | 33 ++ config/config.go | 46 ++ config/setup.go | 147 ++++++ config/setup_test.go | 596 ++++++++++++++++++++++ example_config.toml | 32 ++ go.mod | 8 +- go.sum | 16 +- integration/main_test.go | 17 +- main.go | 20 +- server/acme/client.go | 26 + server/cache/interface.go | 3 +- server/cache/{setup.go => memory.go} | 2 +- server/certificates/acme_client.go | 11 +- server/certificates/acme_config.go | 34 +- server/certificates/cached_challengers.go | 6 +- server/certificates/certificates.go | 2 +- server/dns/dns.go | 2 +- server/gitea/cache.go | 4 +- server/gitea/client.go | 28 +- server/handler/handler.go | 30 +- server/handler/handler_custom_domain.go | 2 +- server/handler/handler_raw_domain.go | 2 +- server/handler/handler_sub_domain.go | 2 +- server/handler/handler_test.go | 29 +- server/handler/try.go | 4 +- server/startup.go | 141 +++++ server/upstream/domains.go | 2 +- server/upstream/redirects.go | 4 +- server/upstream/upstream.go | 2 +- 37 files changed, 1270 insertions(+), 344 deletions(-) create mode 100644 .env-dev create mode 100644 .vscode/launch.json rename {cmd => cli}/certs.go (92%) rename {cmd => cli}/flags.go (80%) create mode 100644 cli/setup.go delete mode 100644 cmd/main.go delete mode 100644 cmd/setup.go create mode 100644 config/assets/test_config.toml create mode 100644 config/config.go create mode 100644 config/setup.go create mode 100644 config/setup_test.go create mode 100644 example_config.toml create mode 100644 server/acme/client.go rename server/cache/{setup.go => memory.go} (69%) create mode 100644 server/startup.go diff --git a/.env-dev b/.env-dev new file mode 100644 index 0000000..2286005 --- /dev/null +++ b/.env-dev @@ -0,0 +1,11 @@ +ACME_API=https://acme.mock.directory +ACME_ACCEPT_TERMS=true +PAGES_DOMAIN=localhost.mock.directory +RAW_DOMAIN=raw.localhost.mock.directory +PAGES_BRANCHES=pages,master,main +GITEA_ROOT=https://codeberg.org +PORT=4430 +HTTP_PORT=8880 +ENABLE_HTTP_SERVER=true +LOG_LEVEL=trace +ACME_ACCOUNT_CONFIG=integration/acme-account.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..d2cc8d1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch PagesServer", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": ["sqlite", "sqlite_unlock_notify", "netgo"], + "envFile": "${workspaceFolder}/.env-dev" + }, + { + "name": "Launch PagesServer integration test", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/integration/main_test.go", + "args": ["codeberg.org/codeberg/pages/integration/..."], + "buildFlags": ["-tags", "'integration sqlite sqlite_unlock_notify netgo'"] + } + ] +} diff --git a/Justfile b/Justfile index 0b8f814..0bf38a3 100644 --- a/Justfile +++ b/Justfile @@ -4,14 +4,9 @@ TAGS := 'sqlite sqlite_unlock_notify netgo' dev: #!/usr/bin/env bash set -euxo pipefail - export ACME_API=https://acme.mock.directory - export ACME_ACCEPT_TERMS=true - export PAGES_DOMAIN=localhost.mock.directory - export RAW_DOMAIN=raw.localhost.mock.directory - export PORT=4430 - export HTTP_PORT=8880 - export ENABLE_HTTP_SERVER=true - export LOG_LEVEL=trace + set -a # automatically export all variables + source .env-dev + set +a go run -tags '{{TAGS}}' . build: @@ -42,10 +37,10 @@ tool-gofumpt: fi test: - go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... test-run TEST: - go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... integration: go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/... diff --git a/cmd/certs.go b/cli/certs.go similarity index 92% rename from cmd/certs.go rename to cli/certs.go index 6012b6e..76df7f0 100644 --- a/cmd/certs.go +++ b/cli/certs.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "fmt" @@ -26,7 +26,7 @@ var Certs = &cli.Command{ } func listCerts(ctx *cli.Context) error { - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } @@ -53,7 +53,7 @@ func removeCert(ctx *cli.Context) error { domains := ctx.Args().Slice() - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } diff --git a/cmd/flags.go b/cli/flags.go similarity index 80% rename from cmd/flags.go rename to cli/flags.go index 7ac94e6..097cf4f 100644 --- a/cmd/flags.go +++ b/cli/flags.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "github.com/urfave/cli/v2" @@ -29,26 +29,35 @@ var ( Name: "gitea-root", Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", EnvVars: []string{"GITEA_ROOT"}, - Value: "https://codeberg.org", }, // GiteaApiToken specifies an api token for the Gitea instance &cli.StringFlag{ Name: "gitea-api-token", Usage: "specifies an api token for the Gitea instance", EnvVars: []string{"GITEA_API_TOKEN"}, - Value: "", }, &cli.BoolFlag{ Name: "enable-lfs-support", Usage: "enable lfs support, require gitea >= v1.17.0 as backend", EnvVars: []string{"ENABLE_LFS_SUPPORT"}, - Value: true, + Value: false, }, &cli.BoolFlag{ Name: "enable-symlink-support", Usage: "follow symlinks if enabled, require gitea >= v1.18.0 as backend", EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, - Value: true, + Value: false, + }, + &cli.StringFlag{ + Name: "default-mime-type", + Usage: "specifies the default mime type for files that don't have a specific mime type.", + EnvVars: []string{"DEFAULT_MIME_TYPE"}, + Value: "application/octet-stream", + }, + &cli.StringSliceFlag{ + Name: "forbidden-mime-types", + Usage: "specifies the forbidden mime types. Use this flag multiple times for multiple mime types.", + EnvVars: []string{"FORBIDDEN_MIME_TYPES"}, }, // ########################### @@ -61,7 +70,6 @@ var ( Name: "pages-domain", Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", EnvVars: []string{"PAGES_DOMAIN"}, - Value: "codeberg.page", }, // RawDomain specifies the domain from which raw repository content shall be served in the following format: // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} @@ -70,7 +78,6 @@ var ( Name: "raw-domain", Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", EnvVars: []string{"RAW_DOMAIN"}, - Value: "raw.codeberg.page", }, // ######################### @@ -98,19 +105,38 @@ var ( Name: "enable-http-server", Usage: "start a http server to redirect to https and respond to http acme challenges", EnvVars: []string{"ENABLE_HTTP_SERVER"}, + Value: false, }, + // Default branches to fetch assets from + &cli.StringSliceFlag{ + Name: "pages-branch", + Usage: "define a branch to fetch assets from. Use this flag multiple times for multiple branches.", + EnvVars: []string{"PAGES_BRANCHES"}, + Value: cli.NewStringSlice("pages"), + }, + + &cli.StringSliceFlag{ + Name: "allowed-cors-domains", + Usage: "specify allowed CORS domains. Use this flag multiple times for multiple domains.", + EnvVars: []string{"ALLOWED_CORS_DOMAINS"}, + }, + &cli.StringSliceFlag{ + Name: "blacklisted-paths", + Usage: "return an error on these url paths.Use this flag multiple times for multiple paths.", + EnvVars: []string{"BLACKLISTED_PATHS"}, + }, + &cli.StringFlag{ Name: "log-level", Value: "warn", Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", EnvVars: []string{"LOG_LEVEL"}, }, - // Default branches to fetch assets from - &cli.StringSliceFlag{ - Name: "pages-branch", - Usage: "define a branch to fetch assets from", - EnvVars: []string{"PAGES_BRANCHES"}, - Value: cli.NewStringSlice("pages"), + &cli.StringFlag{ + Name: "config-file", + Usage: "specify the location of the config file", + Aliases: []string{"config"}, + EnvVars: []string{"CONFIG_FILE"}, }, // ############################ diff --git a/cli/setup.go b/cli/setup.go new file mode 100644 index 0000000..6bbff40 --- /dev/null +++ b/cli/setup.go @@ -0,0 +1,39 @@ +package cli + +import ( + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + "codeberg.org/codeberg/pages/server/database" + "codeberg.org/codeberg/pages/server/version" +) + +func CreatePagesApp() *cli.App { + app := cli.NewApp() + app.Name = "pages-server" + app.Version = version.Version + app.Usage = "pages server" + app.Flags = ServerFlags + app.Commands = []*cli.Command{ + Certs, + } + + return app +} + +func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) + } + + closeFn = func() { + if err := certDB.Close(); err != nil { + log.Error().Err(err) + } + } + + return certDB, closeFn, nil +} diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 683e859..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,150 +0,0 @@ -package cmd - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "os" - "strings" - "time" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/handler" -) - -// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. -// TODO: make it a flag -var AllowedCorsDomains = []string{ - "fonts.codeberg.org", - "design.codeberg.org", -} - -// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. -// TODO: Make it a flag too -var BlacklistedPaths = []string{ - "/.well-known/acme-challenge/", -} - -// Serve sets up and starts the web server. -func Serve(ctx *cli.Context) error { - // Initialize the logger. - logLevel, err := zerolog.ParseLevel(ctx.String("log-level")) - if err != nil { - return err - } - log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) - - giteaRoot := ctx.String("gitea-root") - giteaAPIToken := ctx.String("gitea-api-token") - rawDomain := ctx.String("raw-domain") - defaultBranches := ctx.StringSlice("pages-branch") - mainDomainSuffix := ctx.String("pages-domain") - listeningHost := ctx.String("host") - listeningSSLPort := ctx.Uint("port") - listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) - listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port")) - enableHTTPServer := ctx.Bool("enable-http-server") - - allowedCorsDomains := AllowedCorsDomains - if rawDomain != "" { - allowedCorsDomains = append(allowedCorsDomains, rawDomain) - } - - // Make sure MainDomain has a trailing dot - if !strings.HasPrefix(mainDomainSuffix, ".") { - mainDomainSuffix = "." + mainDomainSuffix - } - - if len(defaultBranches) == 0 { - return fmt.Errorf("no default branches set (PAGES_BRANCHES)") - } - - // Init ssl cert database - certDB, closeFn, err := openCertDB(ctx) - if err != nil { - return err - } - defer closeFn() - - keyCache := cache.NewKeyValueCache() - challengeCache := cache.NewKeyValueCache() - // canonicalDomainCache stores canonical domains - canonicalDomainCache := cache.NewKeyValueCache() - // dnsLookupCache stores DNS lookups for custom domains - dnsLookupCache := cache.NewKeyValueCache() - // redirectsCache stores redirects in _redirects files - redirectsCache := cache.NewKeyValueCache() - // clientResponseCache stores responses from the Gitea server - clientResponseCache := cache.NewKeyValueCache() - - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) - if err != nil { - return fmt.Errorf("could not create new gitea client: %v", err) - } - - acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache) - if err != nil { - return err - } - - if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil { - return err - } - - // Create listener for SSL connections - log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) - listener, err := net.Listen("tcp", listeningSSLAddress) - if err != nil { - return fmt.Errorf("couldn't create listener: %v", err) - } - - // Setup listener for SSL connections - listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, - giteaClient, - acmeClient, - defaultBranches[0], - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, - certDB)) - - interval := 12 * time.Hour - certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) - defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB) - - if enableHTTPServer { - // Create handler for http->https redirect and http acme challenges - httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort) - - // Create listener for http and start listening - go func() { - log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) - err := http.ListenAndServe(listeningHTTPAddress, httpHandler) - if err != nil { - log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") - } - }() - } - - // Create ssl handler based on settings - sslHandler := handler.Handler(mainDomainSuffix, rawDomain, - giteaClient, - BlacklistedPaths, allowedCorsDomains, - defaultBranches, - dnsLookupCache, canonicalDomainCache, redirectsCache) - - // Start the ssl listener - log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) - if err := http.Serve(listener, sslHandler); err != nil { - log.Panic().Err(err).Msg("Couldn't start fastServer") - } - - return nil -} diff --git a/cmd/setup.go b/cmd/setup.go deleted file mode 100644 index cde4bc9..0000000 --- a/cmd/setup.go +++ /dev/null @@ -1,64 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/database" -) - -var ErrAcmeMissConfig = errors.New("ACME client has wrong config") - -func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { - certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to database: %w", err) - } - - closeFn = func() { - if err := certDB.Close(); err != nil { - log.Error().Err(err) - } - } - - return certDB, closeFn, nil -} - -func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) { - acmeAPI := ctx.String("acme-api-endpoint") - acmeMail := ctx.String("acme-email") - acmeEabHmac := ctx.String("acme-eab-hmac") - acmeEabKID := ctx.String("acme-eab-kid") - acmeAcceptTerms := ctx.Bool("acme-accept-terms") - dnsProvider := ctx.String("dns-provider") - acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") - acmeAccountConf := ctx.String("acme-account-config") - - // check config - if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { - return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) - } - if acmeEabHmac != "" && acmeEabKID == "" { - return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) - } else if acmeEabHmac == "" && acmeEabKID != "" { - return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) - } - - return certificates.NewAcmeClient( - acmeAccountConf, - acmeAPI, - acmeMail, - acmeEabHmac, - acmeEabKID, - dnsProvider, - acmeAcceptTerms, - enableHTTPServer, - acmeUseRateLimits, - challengeCache, - ) -} diff --git a/config/assets/test_config.toml b/config/assets/test_config.toml new file mode 100644 index 0000000..6a2f0d0 --- /dev/null +++ b/config/assets/test_config.toml @@ -0,0 +1,33 @@ +logLevel = 'trace' + +[server] +host = '127.0.0.1' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org'] +blacklistedPaths = ['do/not/use'] + +[gitea] +root = 'codeberg.org' +token = 'XXXXXXXX' +lfsEnabled = true +followSymlinks = true +defaultMimeType = "application/wasm" +forbiddenMimeTypes = ["text/html"] + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'a@b.c' +apiEndpoint = 'https://example.com' +acceptTerms = false +useRateLimits = true +eab_hmac = 'asdf' +eab_kid = 'qwer' +dnsProvider = 'cloudflare.com' +accountConfigFile = 'nope' diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..6cb972b --- /dev/null +++ b/config/config.go @@ -0,0 +1,46 @@ +package config + +type Config struct { + LogLevel string `default:"warn"` + Server ServerConfig + Gitea GiteaConfig + Database DatabaseConfig + ACME ACMEConfig +} + +type ServerConfig struct { + Host string `default:"[::]"` + Port uint16 `default:"443"` + HttpPort uint16 `default:"80"` + HttpServerEnabled bool `default:"true"` + MainDomain string + RawDomain string + PagesBranches []string + AllowedCorsDomains []string + BlacklistedPaths []string +} + +type GiteaConfig struct { + Root string + Token string + LFSEnabled bool `default:"false"` + FollowSymlinks bool `default:"false"` + DefaultMimeType string `default:"application/octet-stream"` + ForbiddenMimeTypes []string +} + +type DatabaseConfig struct { + Type string `default:"sqlite3"` + Conn string `default:"certs.sqlite"` +} + +type ACMEConfig struct { + Email string + APIEndpoint string `default:"https://acme-v02.api.letsencrypt.org/directory"` + AcceptTerms bool `default:"false"` + UseRateLimits bool `default:"true"` + EAB_HMAC string + EAB_KID string + DNSProvider string + AccountConfigFile string `default:"acme-account.json"` +} diff --git a/config/setup.go b/config/setup.go new file mode 100644 index 0000000..e774084 --- /dev/null +++ b/config/setup.go @@ -0,0 +1,147 @@ +package config + +import ( + "os" + "path" + + "github.com/creasty/defaults" + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" +) + +var ALWAYS_BLACKLISTED_PATHS = []string{ + "/.well-known/acme-challenge/", +} + +func NewDefaultConfig() Config { + config := Config{} + if err := defaults.Set(&config); err != nil { + panic(err) + } + + // defaults does not support setting arrays from strings + config.Server.PagesBranches = []string{"main", "master", "pages"} + + return config +} + +func ReadConfig(ctx *cli.Context) (*Config, error) { + config := NewDefaultConfig() + // if config is not given as argument return empty config + if !ctx.IsSet("config-file") { + return &config, nil + } + + configFile := path.Clean(ctx.String("config-file")) + + log.Debug().Str("config-file", configFile).Msg("reading config file") + content, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + err = toml.Unmarshal(content, &config) + return &config, err +} + +func MergeConfig(ctx *cli.Context, config *Config) { + if ctx.IsSet("log-level") { + config.LogLevel = ctx.String("log-level") + } + + mergeServerConfig(ctx, &config.Server) + mergeGiteaConfig(ctx, &config.Gitea) + mergeDatabaseConfig(ctx, &config.Database) + mergeACMEConfig(ctx, &config.ACME) +} + +func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { + if ctx.IsSet("host") { + config.Host = ctx.String("host") + } + if ctx.IsSet("port") { + config.Port = uint16(ctx.Uint("port")) + } + if ctx.IsSet("http-port") { + config.HttpPort = uint16(ctx.Uint("http-port")) + } + if ctx.IsSet("enable-http-server") { + config.HttpServerEnabled = ctx.Bool("enable-http-server") + } + if ctx.IsSet("pages-domain") { + config.MainDomain = ctx.String("pages-domain") + } + if ctx.IsSet("raw-domain") { + config.RawDomain = ctx.String("raw-domain") + } + if ctx.IsSet("pages-branch") { + config.PagesBranches = ctx.StringSlice("pages-branch") + } + if ctx.IsSet("allowed-cors-domains") { + config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains") + } + if ctx.IsSet("blacklisted-paths") { + config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths") + } + + // add the paths that should always be blacklisted + config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) +} + +func mergeGiteaConfig(ctx *cli.Context, config *GiteaConfig) { + if ctx.IsSet("gitea-root") { + config.Root = ctx.String("gitea-root") + } + if ctx.IsSet("gitea-api-token") { + config.Token = ctx.String("gitea-api-token") + } + if ctx.IsSet("enable-lfs-support") { + config.LFSEnabled = ctx.Bool("enable-lfs-support") + } + if ctx.IsSet("enable-symlink-support") { + config.FollowSymlinks = ctx.Bool("enable-symlink-support") + } + if ctx.IsSet("default-mime-type") { + config.DefaultMimeType = ctx.String("default-mime-type") + } + if ctx.IsSet("forbidden-mime-types") { + config.ForbiddenMimeTypes = ctx.StringSlice("forbidden-mime-types") + } +} + +func mergeDatabaseConfig(ctx *cli.Context, config *DatabaseConfig) { + if ctx.IsSet("db-type") { + config.Type = ctx.String("db-type") + } + if ctx.IsSet("db-conn") { + config.Conn = ctx.String("db-conn") + } +} + +func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { + if ctx.IsSet("acme-email") { + config.Email = ctx.String("acme-email") + } + if ctx.IsSet("acme-api-endpoint") { + config.APIEndpoint = ctx.String("acme-api-endpoint") + } + if ctx.IsSet("acme-accept-terms") { + config.AcceptTerms = ctx.Bool("acme-accept-terms") + } + if ctx.IsSet("acme-use-rate-limits") { + config.UseRateLimits = ctx.Bool("acme-use-rate-limits") + } + if ctx.IsSet("acme-eab-hmac") { + config.EAB_HMAC = ctx.String("acme-eab-hmac") + } + if ctx.IsSet("acme-eab-kid") { + config.EAB_KID = ctx.String("acme-eab-kid") + } + if ctx.IsSet("dns-provider") { + config.DNSProvider = ctx.String("dns-provider") + } + if ctx.IsSet("acme-account-config") { + config.AccountConfigFile = ctx.String("acme-account-config") + } +} diff --git a/config/setup_test.go b/config/setup_test.go new file mode 100644 index 0000000..a863e2f --- /dev/null +++ b/config/setup_test.go @@ -0,0 +1,596 @@ +package config + +import ( + "context" + "os" + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" +) + +func runApp(t *testing.T, fn func(*cli.Context) error, args []string) { + app := cmd.CreatePagesApp() + app.Action = fn + + appCtx, appCancel := context.WithCancel(context.Background()) + defer appCancel() + + // os.Args always contains the binary name + args = append([]string{"testing"}, args...) + + err := app.RunContext(appCtx, args) + assert.NoError(t, err) +} + +// fixArrayFromCtx fixes the number of "changed" strings in a string slice according to the number of values in the context. +// This is a workaround because the cli library has a bug where the number of values in the context gets bigger the more tests are run. +func fixArrayFromCtx(ctx *cli.Context, key string, expected []string) []string { + if ctx.IsSet(key) { + ctxSlice := ctx.StringSlice(key) + + if len(ctxSlice) > 1 { + for i := 1; i < len(ctxSlice); i++ { + expected = append([]string{"changed"}, expected...) + } + } + } + + return expected +} + +func readTestConfig() (*Config, error) { + content, err := os.ReadFile("assets/test_config.toml") + if err != nil { + return nil, err + } + + expectedConfig := NewDefaultConfig() + err = toml.Unmarshal(content, &expectedConfig) + if err != nil { + return nil, err + } + + return &expectedConfig, nil +} + +func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + expected := NewDefaultConfig() + assert.Equal(t, &expected, cfg) + + return err + }, + []string{}, + ) +} + +func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{"--config-file", "assets/test_config.toml"}, + ) +} + +func TestValuesReadFromConfigFileShouldBeOverwrittenByArgs(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + MergeConfig(ctx, cfg) + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + expectedConfig.LogLevel = "debug" + expectedConfig.Gitea.Root = "not-codeberg.org" + expectedConfig.ACME.AcceptTerms = true + expectedConfig.Server.Host = "172.17.0.2" + expectedConfig.Server.BlacklistedPaths = append(expectedConfig.Server.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--config-file", "assets/test_config.toml", + "--log-level", "debug", + "--gitea-root", "not-codeberg.org", + "--acme-accept-terms", + "--host", "172.17.0.2", + }, + ) +} + +func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &Config{ + LogLevel: "original", + Server: ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + }, + Gitea: GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + }, + Database: DatabaseConfig{ + Type: "original", + Conn: "original", + }, + ACME: ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + }, + } + + MergeConfig(ctx, cfg) + + expectedConfig := &Config{ + LogLevel: "changed", + Server: ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + PagesBranches: []string{"changed"}, + AllowedCorsDomains: []string{"changed"}, + BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...), + }, + Gitea: GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: []string{"changed"}, + }, + Database: DatabaseConfig{ + Type: "changed", + Conn: "changed", + }, + ACME: ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + }, + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--log-level", "changed", + // Server + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--pages-branch", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + // Gitea + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + // Database + "--db-type", "changed", + "--db-conn", "changed", + // ACME + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{} + mergeServerConfig(ctx, cfg) + + expected := ALWAYS_BLACKLISTED_PATHS + assert.Equal(t, expected, cfg.BlacklistedPaths) + + return nil + }, + []string{}, + ) +} + +func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + for range []uint8{0, 1} { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + mergeServerConfig(ctx, cfg) + + expectedConfig := &ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), + BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + }, + ) + } +} + +func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ServerConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }}, + {args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }}, + {args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }}, + {args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }}, + {args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }}, + {args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }}, + {args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }}, + {args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }}, + {args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + expectedConfig.PagesBranches = fixArrayFromCtx(ctx, "pages-branch", expectedConfig.PagesBranches) + expectedConfig.AllowedCorsDomains = fixArrayFromCtx(ctx, "allowed-cors-domains", expectedConfig.AllowedCorsDomains) + expectedConfig.BlacklistedPaths = fixArrayFromCtx(ctx, "blacklisted-paths", expectedConfig.BlacklistedPaths) + + mergeServerConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeGiteaConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + mergeGiteaConfig(ctx, cfg) + + expectedConfig := &GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: fixArrayFromCtx(ctx, "forbidden-mime-types", []string{"changed"}), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + }, + ) +} + +func TestMergeGiteaConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*GiteaConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--gitea-root", "changed"}, callback: func(gc *GiteaConfig) { gc.Root = "changed" }}, + {args: []string{"--gitea-api-token", "changed"}, callback: func(gc *GiteaConfig) { gc.Token = "changed" }}, + {args: []string{"--enable-lfs-support"}, callback: func(gc *GiteaConfig) { gc.LFSEnabled = true }}, + {args: []string{"--enable-symlink-support"}, callback: func(gc *GiteaConfig) { gc.FollowSymlinks = true }}, + {args: []string{"--default-mime-type", "changed"}, callback: func(gc *GiteaConfig) { gc.DefaultMimeType = "changed" }}, + {args: []string{"--forbidden-mime-types", "changed"}, callback: func(gc *GiteaConfig) { gc.ForbiddenMimeTypes = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeGiteaConfig(ctx, &cfg) + + expectedConfig.ForbiddenMimeTypes = fixArrayFromCtx(ctx, "forbidden-mime-types", expectedConfig.ForbiddenMimeTypes) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeDatabaseConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &DatabaseConfig{ + Type: "original", + Conn: "original", + } + + mergeDatabaseConfig(ctx, cfg) + + expectedConfig := &DatabaseConfig{ + Type: "changed", + Conn: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--db-type", "changed", + "--db-conn", "changed", + }, + ) +} + +func TestMergeDatabaseConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*DatabaseConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--db-type", "changed"}, callback: func(gc *DatabaseConfig) { gc.Type = "changed" }}, + {args: []string{"--db-conn", "changed"}, callback: func(gc *DatabaseConfig) { gc.Conn = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := DatabaseConfig{ + Type: "original", + Conn: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeDatabaseConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + mergeACMEConfig(ctx, cfg) + + expectedConfig := &ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ACMEConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--acme-email", "changed"}, callback: func(gc *ACMEConfig) { gc.Email = "changed" }}, + {args: []string{"--acme-api-endpoint", "changed"}, callback: func(gc *ACMEConfig) { gc.APIEndpoint = "changed" }}, + {args: []string{"--acme-accept-terms"}, callback: func(gc *ACMEConfig) { gc.AcceptTerms = true }}, + {args: []string{"--acme-use-rate-limits"}, callback: func(gc *ACMEConfig) { gc.UseRateLimits = true }}, + {args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }}, + {args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }}, + {args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }}, + {args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeACMEConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} diff --git a/example_config.toml b/example_config.toml new file mode 100644 index 0000000..30e77c4 --- /dev/null +++ b/example_config.toml @@ -0,0 +1,32 @@ +logLevel = 'debug' + +[server] +host = '[::]' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +pagesBranches = ["pages"] +allowedCorsDomains = [] +blacklistedPaths = [] + +[gitea] +root = 'https://codeberg.org' +token = 'ASDF1234' +lfsEnabled = true +followSymlinks = true + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'noreply@example.email' +apiEndpoint = 'https://acme-v02.api.letsencrypt.org/directory' +acceptTerms = false +useRateLimits = false +eab_hmac = '' +eab_kid = '' +dnsProvider = '' +accountConfigFile = 'acme-account.json' diff --git a/go.mod b/go.mod index eba292e..47e1e71 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 github.com/microcosm-cc/bluemonday v1.0.26 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.3.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb xorm.io/xorm v1.3.2 @@ -44,6 +45,7 @@ require ( github.com/cloudflare/cloudflare-go v0.20.0 // indirect github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect @@ -113,7 +115,7 @@ require ( github.com/softlayer/softlayer-go v1.0.3 // indirect github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect @@ -135,6 +137,6 @@ require ( gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect xorm.io/builder v0.3.12 // indirect ) diff --git a/go.sum b/go.sum index 7ea8b78..1a10599 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -570,6 +572,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -680,14 +684,19 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1079,8 +1088,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration/main_test.go b/integration/main_test.go index 6566f78..86fd9d3 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -10,9 +10,10 @@ import ( "testing" "time" - "codeberg.org/codeberg/pages/cmd" - "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func TestMain(m *testing.M) { @@ -32,10 +33,7 @@ func TestMain(m *testing.M) { } func startServer(ctx context.Context) error { - args := []string{ - "--verbose", - "--acme-accept-terms", "true", - } + args := []string{"integration"} setEnvIfNotSet("ACME_API", "https://acme.mock.directory") setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") @@ -44,10 +42,15 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("HTTP_PORT", "8880") setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") setEnvIfNotSet("DB_TYPE", "sqlite3") + setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org") + setEnvIfNotSet("LOG_LEVEL", "trace") + setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true") + setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true") + setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json") app := cli.NewApp() app.Name = "pages-server" - app.Action = cmd.Serve + app.Action = server.Serve app.Flags = cmd.ServerFlags go func() { diff --git a/main.go b/main.go index 6c1d0cc..87e21f3 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,21 @@ package main import ( - "fmt" "os" _ "github.com/joho/godotenv/autoload" - "github.com/urfave/cli/v2" + "github.com/rs/zerolog/log" - "codeberg.org/codeberg/pages/cmd" - "codeberg.org/codeberg/pages/server/version" + "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func main() { - app := cli.NewApp() - app.Name = "pages-server" - app.Version = version.Version - app.Usage = "pages server" - app.Action = cmd.Serve - app.Flags = cmd.ServerFlags - app.Commands = []*cli.Command{ - cmd.Certs, - } + app := cli.CreatePagesApp() + app.Action = server.Serve if err := app.Run(os.Args); err != nil { - _, _ = fmt.Fprintln(os.Stderr, err) + log.Error().Err(err).Msg("A fatal error occurred") os.Exit(1) } } diff --git a/server/acme/client.go b/server/acme/client.go new file mode 100644 index 0000000..38e2785 --- /dev/null +++ b/server/acme/client.go @@ -0,0 +1,26 @@ +package acme + +import ( + "errors" + "fmt" + + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" +) + +var ErrAcmeMissConfig = errors.New("ACME client has wrong config") + +func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) { + // check config + if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + } + if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" { + return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) + } else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" { + return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) + } + + return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache) +} diff --git a/server/cache/interface.go b/server/cache/interface.go index 2952b29..b3412cc 100644 --- a/server/cache/interface.go +++ b/server/cache/interface.go @@ -2,7 +2,8 @@ package cache import "time" -type SetGetKey interface { +// ICache is an interface that defines how the pages server interacts with the cache. +type ICache interface { Set(key string, value interface{}, ttl time.Duration) error Get(key string) (interface{}, bool) Remove(key string) diff --git a/server/cache/setup.go b/server/cache/memory.go similarity index 69% rename from server/cache/setup.go rename to server/cache/memory.go index a5928b0..093696f 100644 --- a/server/cache/setup.go +++ b/server/cache/memory.go @@ -2,6 +2,6 @@ package cache import "github.com/OrlovEvgeny/go-mcache" -func NewKeyValueCache() SetGetKey { +func NewInMemoryCache() ICache { return mcache.New() } diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index ba83e50..d53e854 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -10,6 +10,7 @@ import ( "github.com/reugn/equalizer" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" ) @@ -28,8 +29,8 @@ type AcmeClient struct { acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket } -func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) { - acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) +func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) { + acmeConfig, err := setupAcmeConfig(cfg) if err != nil { return nil, err } @@ -54,7 +55,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { - if dnsProvider == "" { + if cfg.DNSProvider == "" { // using mock server, don't use wildcard certs err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { @@ -62,7 +63,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, } } else { // use DNS-Challenge https://go-acme.github.io/lego/dns/ - provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) + provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider) if err != nil { return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err) } @@ -76,7 +77,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, legoClient: acmeClient, dnsChallengerLegoClient: mainDomainAcmeClient, - acmeUseRateLimits: acmeUseRateLimits, + acmeUseRateLimits: cfg.UseRateLimits, obtainLocks: sync.Map{}, diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go index 12ad7c6..2b5151d 100644 --- a/server/certificates/acme_config.go +++ b/server/certificates/acme_config.go @@ -8,6 +8,7 @@ import ( "fmt" "os" + "codeberg.org/codeberg/pages/config" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/lego" "github.com/go-acme/lego/v4/registration" @@ -16,21 +17,27 @@ import ( const challengePath = "/.well-known/acme-challenge/" -func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { +func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) { var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config - if account, err := os.ReadFile(configFile); err == nil { - log.Info().Msgf("found existing acme account config file '%s'", configFile) + if cfg.AccountConfigFile == "" { + return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile) + } + + if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil { + log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile) if err := json.Unmarshal(account, &myAcmeAccount); err != nil { return nil, err } + myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) if err != nil { return nil, err } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 // Validate Config @@ -39,6 +46,7 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate") return nil, fmt.Errorf("acme config validation failed: %w", err) } + return myAcmeConfig, nil } else if !os.IsNotExist(err) { return nil, err @@ -51,20 +59,20 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri return nil, err } myAcmeAccount = AcmeAccount{ - Email: acmeMail, + Email: cfg.Email, Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { // accept terms & log in to EAB - if acmeEabKID == "" || acmeEabHmac == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) + if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms}) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") } else { @@ -72,9 +80,9 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri } } else { reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: acmeAcceptTerms, - Kid: acmeEabKID, - HmacEncoded: acmeEabHmac, + TermsOfServiceAgreed: cfg.AcceptTerms, + Kid: cfg.EAB_KID, + HmacEncoded: cfg.EAB_HMAC, }) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") @@ -89,8 +97,8 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits") select {} } - log.Info().Msgf("new acme account created. write to config file '%s'", configFile) - err = os.WriteFile(configFile, acmeAccountJSON, 0o600) + log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile) + err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600) if err != nil { log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits") select {} diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index bc9ea67..39439fb 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -15,7 +15,7 @@ import ( ) type AcmeTLSChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeTLSChallengeProvider match Provider interface @@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { } type AcmeHTTPChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeHTTPChallengeProvider match Provider interface @@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc { +func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc { // handle custom-ssl-ports to be added on https redirects portPart := "" if sslPort != 443 { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 4fb99fa..b638755 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -31,7 +31,7 @@ func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, firstDefaultBranch string, - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, ) *tls.Config { return &tls.Config{ diff --git a/server/dns/dns.go b/server/dns/dns.go index c11b278..970f0c0 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -15,7 +15,7 @@ var defaultPagesRepo = "pages" // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { +func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error diff --git a/server/gitea/cache.go b/server/gitea/cache.go index af61edf..267c3d8 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -74,7 +74,7 @@ type writeCacheReader struct { buffer *bytes.Buffer rileResponse *FileResponse cacheKey string - cache cache.SetGetKey + cache cache.ICache hasError bool } @@ -99,7 +99,7 @@ func (t *writeCacheReader) Close() error { return t.originalReader.Close() } -func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { +func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser { if r == nil || cache == nil || cacheKey == "" { log.Error().Msg("could not create CacheReader") return nil diff --git a/server/gitea/client.go b/server/gitea/client.go index f3bda54..42cf065 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/version" ) @@ -44,7 +45,7 @@ const ( type Client struct { sdkClient *gitea.Client - responseCache cache.SetGetKey + responseCache cache.ICache giteaRoot string @@ -55,24 +56,21 @@ type Client struct { defaultMimeType string } -func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { - rootURL, err := url.Parse(giteaRoot) +func NewClient(cfg config.GiteaConfig, respCache cache.ICache) (*Client, error) { + rootURL, err := url.Parse(cfg.Root) if err != nil { return nil, err } - giteaRoot = strings.Trim(rootURL.String(), "/") + giteaRoot := strings.Trim(rootURL.String(), "/") stdClient := http.Client{Timeout: 10 * time.Second} - // TODO: pass down - var ( - forbiddenMimeTypes map[string]bool - defaultMimeType string - ) - - if forbiddenMimeTypes == nil { - forbiddenMimeTypes = make(map[string]bool) + forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes)) + for _, mimeType := range cfg.ForbiddenMimeTypes { + forbiddenMimeTypes[mimeType] = true } + + defaultMimeType := cfg.DefaultMimeType if defaultMimeType == "" { defaultMimeType = "application/octet-stream" } @@ -80,7 +78,7 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo sdk, err := gitea.NewClient( giteaRoot, gitea.SetHTTPClient(&stdClient), - gitea.SetToken(giteaAPIToken), + gitea.SetToken(cfg.Token), gitea.SetUserAgent("pages-server/"+version.Version), ) @@ -90,8 +88,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo giteaRoot: giteaRoot, - followSymlinks: followSymlinks, - supportLFS: supportLFS, + followSymlinks: cfg.FollowSymlinks, + supportLFS: cfg.LFSEnabled, forbiddenMimeTypes: forbiddenMimeTypes, defaultMimeType: defaultMimeType, diff --git a/server/handler/handler.go b/server/handler/handler.go index 7da5d39..96788e4 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" @@ -19,11 +20,10 @@ const ( ) // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain string, +func Handler( + cfg config.ServerConfig, giteaClient *gitea.Client, - blacklistedPaths, allowedCorsDomains []string, - defaultPagesBranches []string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() @@ -39,8 +39,8 @@ func Handler(mainDomainSuffix, rawDomain string, trimmedHost := ctx.TrimHostPort() - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + // Add HSTS for RawDomain and MainDomain + if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" { ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) } @@ -62,7 +62,7 @@ func Handler(mainDomainSuffix, rawDomain string, } // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range blacklistedPaths { + for _, blacklistedPath := range cfg.BlacklistedPaths { if strings.HasPrefix(ctx.Path(), blacklistedPath) { html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden) return @@ -71,7 +71,7 @@ func Handler(mainDomainSuffix, rawDomain string, // Allow CORS for specified domains allowCors := false - for _, allowedCorsDomain := range allowedCorsDomains { + for _, allowedCorsDomain := range cfg.AllowedCorsDomains { if strings.EqualFold(trimmedHost, allowedCorsDomain) { allowCors = true break @@ -85,28 +85,28 @@ func Handler(mainDomainSuffix, rawDomain string, // Prepare request information to Gitea pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { + if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) { log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) - } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + } else if strings.HasSuffix(trimmedHost, cfg.MainDomain) { log.Debug().Msg("subdomain request detected") handleSubDomain(log, ctx, giteaClient, - mainDomainSuffix, - defaultPagesBranches, + cfg.MainDomain, + cfg.PagesBranches, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) } else { log.Debug().Msg("custom domain request detected") handleCustomDomain(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, - defaultPagesBranches[0], + cfg.PagesBranches[0], dnsLookupCache, canonicalDomainCache, redirectsCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 8742be4..82953f9 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g trimmedHost string, pathElements []string, firstDefaultBranch string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from custom domains targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index caa8209..f48e8e4 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie mainDomainSuffix string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve raw content from RawDomain log.Debug().Msg("raw domain") diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 6c14393..806fe7f 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite defaultPagesBranches []string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from subdomains of MainDomainSuffix log.Debug().Msg("main domain suffix") diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index d04ebda..4cb859a 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -6,23 +6,30 @@ import ( "testing" "time" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" "github.com/rs/zerolog/log" ) func TestHandlerPerformance(t *testing.T) { - giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false) - testHandler := Handler( - "codeberg.page", "raw.codeberg.org", - giteaClient, - []string{"/.well-known/acme-challenge/"}, - []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, - []string{"pages"}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - ) + cfg := config.GiteaConfig{ + Root: "https://codeberg.org", + Token: "", + LFSEnabled: false, + FollowSymlinks: false, + } + giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache()) + serverCfg := config.ServerConfig{ + MainDomain: "codeberg.page", + RawDomain: "raw.codeberg.page", + BlacklistedPaths: []string{ + "/.well-known/acme-challenge/", + }, + AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, + PagesBranches: []string{"pages"}, + } + testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache()) testCase := func(uri string, status int) { t.Run(uri, func(t *testing.T) { diff --git a/server/handler/try.go b/server/handler/try.go index bb79dce..145b1a9 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -17,8 +17,8 @@ import ( func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost string, options *upstream.Options, - canonicalDomainCache cache.SetGetKey, - redirectsCache cache.SetGetKey, + canonicalDomainCache cache.ICache, + redirectsCache cache.ICache, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { diff --git a/server/startup.go b/server/startup.go new file mode 100644 index 0000000..ffdabb7 --- /dev/null +++ b/server/startup.go @@ -0,0 +1,141 @@ +package server + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/acme" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/handler" +) + +// Serve sets up and starts the web server. +func Serve(ctx *cli.Context) error { + // initialize logger with Trace, overridden later with actual level + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel) + + cfg, err := config.ReadConfig(ctx) + if err != nil { + log.Error().Err(err).Msg("could not read config") + } + + config.MergeConfig(ctx, cfg) + + // Initialize the logger. + logLevel, err := zerolog.ParseLevel(cfg.LogLevel) + if err != nil { + return err + } + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) + + foo, _ := json.Marshal(cfg) + log.Trace().RawJSON("config", foo).Msg("starting server with config") + + listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort) + + if cfg.Server.RawDomain != "" { + cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain) + } + + // Make sure MainDomain has a leading dot + if !strings.HasPrefix(cfg.Server.MainDomain, ".") { + // TODO make this better + cfg.Server.MainDomain = "." + cfg.Server.MainDomain + } + + if len(cfg.Server.PagesBranches) == 0 { + return fmt.Errorf("no default branches set (PAGES_BRANCHES)") + } + + // Init ssl cert database + certDB, closeFn, err := cmd.OpenCertDB(ctx) + if err != nil { + return err + } + defer closeFn() + + keyCache := cache.NewInMemoryCache() + challengeCache := cache.NewInMemoryCache() + // canonicalDomainCache stores canonical domains + canonicalDomainCache := cache.NewInMemoryCache() + // dnsLookupCache stores DNS lookups for custom domains + dnsLookupCache := cache.NewInMemoryCache() + // redirectsCache stores redirects in _redirects files + redirectsCache := cache.NewInMemoryCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewInMemoryCache() + + giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache) + if err != nil { + return fmt.Errorf("could not create new gitea client: %v", err) + } + + acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache) + if err != nil { + return err + } + + if err := certificates.SetupMainDomainCertificates(cfg.Server.MainDomain, acmeClient, certDB); err != nil { + return err + } + + // Create listener for SSL connections + log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) + listener, err := net.Listen("tcp", listeningSSLAddress) + if err != nil { + return fmt.Errorf("couldn't create listener: %v", err) + } + + // Setup listener for SSL connections + listener = tls.NewListener(listener, certificates.TLSConfig( + cfg.Server.MainDomain, + giteaClient, + acmeClient, + cfg.Server.PagesBranches[0], + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, + certDB, + )) + + interval := 12 * time.Hour + certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) + defer cancelCertMaintain() + go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB) + + if cfg.Server.HttpServerEnabled { + // Create handler for http->https redirect and http acme challenges + httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port)) + + // Create listener for http and start listening + go func() { + log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) + err := http.ListenAndServe(listeningHTTPAddress, httpHandler) + if err != nil { + log.Error().Err(err).Msg("Couldn't start HTTP server") + } + }() + } + + // Create ssl handler based on settings + sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache) + + // Start the ssl listener + log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) + + return http.Serve(listener, sslHandler) +} diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 230f268..d53d586 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute const canonicalDomainConfig = ".domains" // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) { +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) { // Check if this request is cached. if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { domains := cachedValue.([]string) diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index ab6c971..dd36a84 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -23,7 +23,7 @@ var redirectsCacheTimeout = 10 * time.Minute const redirectsConfig = "_redirects" // getRedirects returns redirects specified in the _redirects file. -func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect { +func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect { var redirects []Redirect cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch @@ -63,7 +63,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S return redirects } -func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) { +func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) { if len(redirects) > 0 { for _, redirect := range redirects { reqUrl := ctx.Req.RequestURI diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 4a85503..07b6ad2 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -53,7 +53,7 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" { From a6e9510c078758794da5f6a1983b595bfc415de9 Mon Sep 17 00:00:00 2001 From: Hoernschen Date: Mon, 26 Feb 2024 22:21:42 +0000 Subject: [PATCH 3/8] FIX blank internal pages (#164) (#292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hello 👋 since it affected my deployment of the pages server I started to look into the problem of the blank pages and think I found a solution for it: 1. There is no check if the file response is empty, neither in cache retrieval nor in writing of a cache. Also the provided method for checking for empty responses had a bug. 2. I identified the redirect response to be the issue here. There is a cache write with the full cache key (e. g. rawContent/user/repo|branch|route/index.html) happening in the handling of the redirect response. But the written body here is empty. In the triggered request from the redirect response the server then finds a cache item to the key and serves the empty body. A quick fix is the check for empty file responses mentioned in 1. 3. The decision to redirect the user comes quite far down in the upstream function. Before that happens a lot of stuff that may not be important since after the redirect response comes a new request anyway. Also, I suspect that this causes the caching problem because there is a request to the forge server and its error handling with some recursions happening before. I propose to move two of the redirects before "Preparing" 4. The recursion in the upstream function makes it difficult to understand what is actually happening. I added some more logging to have an easier time with that. 5. I changed the default behaviour to append a trailing slash to the path to true. In my tested scenarios it happened anyway. This way there is no recursion happening before the redirect. I am not developing in go frequently and rarely contribute to open source -> so feedback of all kind is appreciated closes #164 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/292 Reviewed-by: 6543 <6543@obermui.de> Reviewed-by: crapStone Co-authored-by: Hoernschen Co-committed-by: Hoernschen --- server/gitea/cache.go | 25 +++++++++++++++++-------- server/gitea/client.go | 19 +++++++++++-------- server/handler/handler.go | 1 + server/upstream/upstream.go | 12 ++++++++++++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 267c3d8..4ecc5e0 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -39,7 +39,7 @@ type FileResponse struct { } func (f FileResponse) IsEmpty() bool { - return len(f.Body) != 0 + return len(f.Body) == 0 } func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) { @@ -72,13 +72,14 @@ type BranchTimestamp struct { type writeCacheReader struct { originalReader io.ReadCloser buffer *bytes.Buffer - rileResponse *FileResponse + fileResponse *FileResponse cacheKey string cache cache.ICache hasError bool } func (t *writeCacheReader) Read(p []byte) (n int, err error) { + log.Trace().Msgf("[cache] read %q", t.cacheKey) n, err = t.originalReader.Read(p) if err != nil && err != io.EOF { log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey) @@ -90,12 +91,20 @@ func (t *writeCacheReader) Read(p []byte) (n int, err error) { } func (t *writeCacheReader) Close() error { - if !t.hasError { - fc := *t.rileResponse - fc.Body = t.buffer.Bytes() - _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + doWrite := !t.hasError + fc := *t.fileResponse + fc.Body = t.buffer.Bytes() + if fc.IsEmpty() { + log.Trace().Msg("[cache] file response is empty") + doWrite = false } - log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError) + if doWrite { + err := t.cache.Set(t.cacheKey, fc, fileCacheTimeout) + if err != nil { + log.Trace().Err(err).Msgf("[cache] writer for %q has returned an error", t.cacheKey) + } + } + log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, doWrite) return t.originalReader.Close() } @@ -108,7 +117,7 @@ func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cac return &writeCacheReader{ originalReader: r, buffer: bytes.NewBuffer(make([]byte, 0)), - rileResponse: &f, + fileResponse: &f, cache: cache, cacheKey: cacheKey, } diff --git a/server/gitea/client.go b/server/gitea/client.go index 42cf065..4f7eaa6 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -112,26 +112,27 @@ func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource str func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) { cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource) log := log.With().Str("cache_key", cacheKey).Logger() - + log.Trace().Msg("try file in cache") // handle if cache entry exist if cache, ok := client.responseCache.Get(cacheKey); ok { cache := cache.(FileResponse) cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey) // TODO: check against some timestamp mismatch?!? if cache.Exists { + log.Debug().Msg("[cache] exists") if cache.IsSymlink { linkDest := string(cache.Body) log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest) return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) - } else { - log.Debug().Msg("[cache] return bytes") + } else if !cache.IsEmpty() { + log.Debug().Msgf("[cache] return %d bytes", len(cache.Body)) return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil + } else if cache.IsEmpty() { + log.Debug().Msg("[cache] is empty") } - } else { - return nil, cachedHeader, cachedStatusCode, ErrorNotFound } } - + log.Trace().Msg("file not in cache") // not in cache, open reader via gitea api reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) if resp != nil { @@ -155,12 +156,14 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str linkDest = path.Join(path.Dir(resource), linkDest) // we store symlink not content to reduce duplicates in cache - if err := client.responseCache.Set(cacheKey, FileResponse{ + fileResponse := FileResponse{ Exists: true, IsSymlink: true, Body: []byte(linkDest), ETag: resp.Header.Get(ETagHeader), - }, fileCacheTimeout); err != nil { + } + log.Trace().Msgf("file response has %d bytes", len(fileResponse.Body)) + if err := client.responseCache.Set(cacheKey, fileResponse, fileCacheTimeout); err != nil { log.Error().Err(err).Msg("[cache] error on cache write") } diff --git a/server/handler/handler.go b/server/handler/handler.go index 96788e4..ffc3400 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -26,6 +26,7 @@ func Handler( dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { + log.Debug().Msg("\n----------------------------------------------------------") log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() ctx := context.New(w, req) diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 07b6ad2..d9c131e 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -56,6 +56,8 @@ type Options struct { func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() + log.Debug().Msg("Start") + if o.TargetOwner == "" || o.TargetRepo == "" { html.ReturnErrorPage(ctx, "forge client: either repo owner or name info is missing", http.StatusBadRequest) return true @@ -104,13 +106,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // Handle not found error if err != nil && errors.Is(err, gitea.ErrorNotFound) { + log.Debug().Msg("Handling not found error") // Get and match redirects redirects := o.getRedirects(giteaClient, redirectsCache) if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) { + log.Trace().Msg("redirect") return true } if o.TryIndexPages { + log.Trace().Msg("try index page") // copy the o struct & try if an index page exists optionsForIndexPages := *o optionsForIndexPages.TryIndexPages = false @@ -121,6 +126,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi return true } } + log.Trace().Msg("try html file with path name") // compatibility fix for GitHub Pages (/example → /example.html) optionsForIndexPages.appendTrailingSlash = false optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" @@ -130,8 +136,11 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi } } + log.Trace().Msg("not found") + ctx.StatusCode = http.StatusNotFound if o.TryIndexPages { + log.Trace().Msg("try not found page") // copy the o struct & try if a not found page exists optionsForNotFoundPages := *o optionsForNotFoundPages.TryIndexPages = false @@ -142,6 +151,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi return true } } + log.Trace().Msg("not found page missing") } return false @@ -173,10 +183,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi // Append trailing slash if missing (for index files), and redirect to fix filenames in general // o.appendTrailingSlash is only true when looking for index pages if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { + log.Trace().Msg("append trailing slash and redirect") ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) return true } if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw { + log.Trace().Msg("remove index.html from path and redirect") ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) return true } From dd6d8bd60fe0e318602042d1ae8129654f4f5186 Mon Sep 17 00:00:00 2001 From: caelandb Date: Tue, 12 Mar 2024 21:50:17 +0000 Subject: [PATCH 4/8] fixed one grammar error. (#297) Just a small grammar change in the README. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/297 Reviewed-by: crapStone Co-authored-by: caelandb Co-committed-by: caelandb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb2a4b9..7193691 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Thank you very much. Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed. run `just dev` -now this pages should work: +now these pages should work: - - From 03881382a4c286e3a7dc351f095f9147e89c2d6d Mon Sep 17 00:00:00 2001 From: Jean-Marie 'Histausse' Mineau Date: Thu, 18 Apr 2024 17:05:20 +0000 Subject: [PATCH 5/8] Add option to disable DNS ACME provider (#290) This PR add the `$NO_DNS_01` option (disabled by default) that removes the DNS ACME provider, and replaces the wildcard certificate by individual certificates obtained using the TLS ACME provider. This option allows an instance to work without having to manage access tokens for the DNS provider. On the flip side, this means that a certificate can be requested for each subdomains. To limit the risk of DOS, the existence of the user/org corresponding to a subdomain is checked before requesting a cert, however, this limitation is not enough for an forge with a high number of users/orgs. Co-authored-by: 6543 <6543@obermui.de> Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/290 Reviewed-by: Moritz Marquardt Co-authored-by: Jean-Marie 'Histausse' Mineau Co-committed-by: Jean-Marie 'Histausse' Mineau --- README.md | 1 + cli/flags.go | 5 +++++ config/config.go | 1 + config/setup.go | 3 +++ config/setup_test.go | 7 ++++++ server/acme/client.go | 4 ++-- server/certificates/acme_client.go | 7 ++---- server/certificates/certificates.go | 30 ++++++++++++++++++++------ server/database/xorm.go | 13 ------------ server/gitea/cache.go | 3 +++ server/gitea/client.go | 33 +++++++++++++++++++++++++++++ server/startup.go | 2 ++ 12 files changed, 83 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 7193691..f47a196 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80. - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard. See for available values & additional environment variables. +- `NO_DNS_01` (default: `false`): Disable the use of ACME DNS. This means that the wildcard certificate is self-signed and all domains and subdomains will have a distinct certificate. Because this may lead to a rate limit from the ACME provider, this option is not recommended for Gitea/Forgejo instances with open registrations or a great number of users/orgs. - `LOG_LEVEL` (default: warn): Set this to specify the level of logging. ## Contributing to the development diff --git a/cli/flags.go b/cli/flags.go index 097cf4f..52e1c1c 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -178,6 +178,11 @@ var ( Usage: "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/", EnvVars: []string{"DNS_PROVIDER"}, }, + &cli.BoolFlag{ + Name: "no-dns-01", + Usage: "Always use individual certificates instead of a DNS-01 wild card certificate", + EnvVars: []string{"NO_DNS_01"}, + }, &cli.StringFlag{ Name: "acme-account-config", Usage: "json file of acme account", diff --git a/config/config.go b/config/config.go index 6cb972b..0146e0f 100644 --- a/config/config.go +++ b/config/config.go @@ -42,5 +42,6 @@ type ACMEConfig struct { EAB_HMAC string EAB_KID string DNSProvider string + NoDNS01 bool `default:"false"` AccountConfigFile string `default:"acme-account.json"` } diff --git a/config/setup.go b/config/setup.go index e774084..6a2aa62 100644 --- a/config/setup.go +++ b/config/setup.go @@ -141,6 +141,9 @@ func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { if ctx.IsSet("dns-provider") { config.DNSProvider = ctx.String("dns-provider") } + if ctx.IsSet("no-dns-01") { + config.NoDNS01 = ctx.Bool("no-dns-01") + } if ctx.IsSet("acme-account-config") { config.AccountConfigFile = ctx.String("acme-account-config") } diff --git a/config/setup_test.go b/config/setup_test.go index a863e2f..1a32740 100644 --- a/config/setup_test.go +++ b/config/setup_test.go @@ -166,6 +166,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T EAB_HMAC: "original", EAB_KID: "original", DNSProvider: "original", + NoDNS01: false, AccountConfigFile: "original", }, } @@ -205,6 +206,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T EAB_HMAC: "changed", EAB_KID: "changed", DNSProvider: "changed", + NoDNS01: true, AccountConfigFile: "changed", }, } @@ -243,6 +245,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T "--acme-eab-hmac", "changed", "--acme-eab-kid", "changed", "--dns-provider", "changed", + "--no-dns-01", "--acme-account-config", "changed", }, ) @@ -517,6 +520,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi EAB_HMAC: "original", EAB_KID: "original", DNSProvider: "original", + NoDNS01: false, AccountConfigFile: "original", } @@ -530,6 +534,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi EAB_HMAC: "changed", EAB_KID: "changed", DNSProvider: "changed", + NoDNS01: true, AccountConfigFile: "changed", } @@ -545,6 +550,7 @@ func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testi "--acme-eab-hmac", "changed", "--acme-eab-kid", "changed", "--dns-provider", "changed", + "--no-dns-01", "--acme-account-config", "changed", }, ) @@ -563,6 +569,7 @@ func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExi {args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }}, {args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }}, {args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }}, + {args: []string{"--no-dns-01"}, callback: func(gc *ACMEConfig) { gc.NoDNS01 = true }}, {args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }}, } diff --git a/server/acme/client.go b/server/acme/client.go index 38e2785..d5c83d0 100644 --- a/server/acme/client.go +++ b/server/acme/client.go @@ -13,8 +13,8 @@ var ErrAcmeMissConfig = errors.New("ACME client has wrong config") func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) { // check config - if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" { - return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + if (!cfg.AcceptTerms || (cfg.DNSProvider == "" && !cfg.NoDNS01)) && cfg.APIEndpoint != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER or $NO_DNS_01, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) } if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" { return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index d53e854..f42fd8f 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -56,11 +56,8 @@ func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { if cfg.DNSProvider == "" { - // using mock server, don't use wildcard certs - err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) - if err != nil { - log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider") - } + // using mock wildcard certs + mainDomainAcmeClient = nil } else { // use DNS-Challenge https://go-acme.github.io/lego/dns/ provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider) diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index b638755..67219dd 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -33,6 +33,8 @@ func TLSConfig(mainDomainSuffix string, firstDefaultBranch string, keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, + noDNS01 bool, + rawDomain string, ) *tls.Config { return &tls.Config{ // check DNS name & get certificate from Let's Encrypt @@ -64,9 +66,24 @@ func TLSConfig(mainDomainSuffix string, targetOwner := "" mayObtainCert := true + if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) { - // deliver default certificate for the main domain (*.codeberg.page) - domain = mainDomainSuffix + if noDNS01 { + // Limit the domains allowed to request a certificate to pages-server domains + // and domains for an existing user of org + if !strings.EqualFold(domain, mainDomainSuffix[1:]) && !strings.EqualFold(domain, rawDomain) { + targetOwner := strings.TrimSuffix(domain, mainDomainSuffix) + owner_exist, err := giteaClient.GiteaCheckIfOwnerExists(targetOwner) + mayObtainCert = owner_exist + if err != nil { + log.Error().Err(err).Msgf("Failed to check '%s' existence on the forge: %s", targetOwner, err) + mayObtainCert = false + } + } + } else { + // deliver default certificate for the main domain (*.codeberg.page) + domain = mainDomainSuffix + } } else { var targetRepo, targetBranch string targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) @@ -199,9 +216,6 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) { name := strings.TrimPrefix(domains[0], "*") - if useDnsProvider && domains[0] != "" && domains[0][0] == '*' { - domains = domains[1:] - } // lock to avoid simultaneous requests _, working := c.obtainLocks.LoadOrStore(name, struct{}{}) @@ -219,7 +233,11 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew defer c.obtainLocks.Delete(name) if acmeClient == nil { - return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) + if useDnsProvider { + return mockCert(domains[0], "DNS ACME client is not defined", mainDomainSuffix, keyDatabase) + } else { + return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase) + } } // request actual cert diff --git a/server/database/xorm.go b/server/database/xorm.go index 217b6d1..63fa39e 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -52,7 +52,6 @@ func (x xDB) Close() error { func (x xDB) Put(domain string, cert *certificate.Resource) error { log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db") - domain = integrationTestReplacements(domain) c, err := toCert(domain, cert) if err != nil { return err @@ -82,7 +81,6 @@ func (x xDB) Get(domain string) (*certificate.Resource, error) { if domain[:1] == "." { domain = "*" + domain } - domain = integrationTestReplacements(domain) cert := new(Cert) log.Trace().Str("domain", domain).Msg("get cert from db") @@ -99,7 +97,6 @@ func (x xDB) Delete(domain string) error { if domain[:1] == "." { domain = "*" + domain } - domain = integrationTestReplacements(domain) log.Trace().Str("domain", domain).Msg("delete cert from db") _, err := x.engine.ID(domain).Delete(new(Cert)) @@ -139,13 +136,3 @@ func supportedDriver(driver string) bool { return false } } - -// integrationTestReplacements is needed because integration tests use a single domain cert, -// while production use a wildcard cert -// TODO: find a better way to handle this -func integrationTestReplacements(domainKey string) string { - if domainKey == "*.localhost.mock.directory" { - return "localhost.mock.directory" - } - return domainKey -} diff --git a/server/gitea/cache.go b/server/gitea/cache.go index 4ecc5e0..97024a1 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -26,6 +26,9 @@ const ( // TODO: move as option into cache interface fileCacheTimeout = 5 * time.Minute + // ownerExistenceCacheTimeout specifies the timeout for the existence of a repo/org + ownerExistenceCacheTimeout = 5 * time.Minute + // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default. fileCacheSizeLimit = int64(1000 * 1000) ) diff --git a/server/gitea/client.go b/server/gitea/client.go index 4f7eaa6..5955bfb 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -28,6 +28,7 @@ const ( branchTimestampCacheKeyPrefix = "branchTime" defaultBranchCacheKeyPrefix = "defaultBranch" rawContentCacheKeyPrefix = "rawContent" + ownerExistenceKeyPrefix = "ownerExist" // pages server PagesCacheIndicatorHeader = "X-Pages-Cache" @@ -266,6 +267,38 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str return branch, nil } +func (client *Client) GiteaCheckIfOwnerExists(owner string) (bool, error) { + cacheKey := fmt.Sprintf("%s/%s", ownerExistenceKeyPrefix, owner) + + if exist, ok := client.responseCache.Get(cacheKey); ok && exist != nil { + return exist.(bool), nil + } + + _, resp, err := client.sdkClient.GetUserInfo(owner) + if resp.StatusCode == http.StatusOK && err == nil { + if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return true, nil + } else if resp.StatusCode != http.StatusNotFound { + return false, err + } + + _, resp, err = client.sdkClient.GetOrg(owner) + if resp.StatusCode == http.StatusOK && err == nil { + if err := client.responseCache.Set(cacheKey, true, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return true, nil + } else if resp.StatusCode != http.StatusNotFound { + return false, err + } + if err := client.responseCache.Set(cacheKey, false, ownerExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msg("[cache] error on cache write") + } + return false, nil +} + func (client *Client) getMimeTypeByExtension(resource string) string { mimeType := mime.TypeByExtension(path.Ext(resource)) mimeTypeSplit := strings.SplitN(mimeType, ";", 2) diff --git a/server/startup.go b/server/startup.go index ffdabb7..149a07d 100644 --- a/server/startup.go +++ b/server/startup.go @@ -110,6 +110,8 @@ func Serve(ctx *cli.Context) error { cfg.Server.PagesBranches[0], keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, certDB, + cfg.ACME.NoDNS01, + cfg.Server.RawDomain, )) interval := 12 * time.Hour From 9ffdc9d4f9406b0cb2e1d07346f65c557a58568a Mon Sep 17 00:00:00 2001 From: Daniel Erat Date: Thu, 18 Apr 2024 21:03:16 +0000 Subject: [PATCH 6/8] Refactor redirect code and add tests (#304) Move repetitive code from Options.matchRedirects into a new Redirect.rewriteURL method and add a new test file. No functional changes are intended; this is in preparation for a later change to address #269. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/304 Reviewed-by: crapStone Co-authored-by: Daniel Erat Co-committed-by: Daniel Erat --- server/upstream/redirects.go | 75 +++++++++++++------------------ server/upstream/redirects_test.go | 37 +++++++++++++++ 2 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 server/upstream/redirects_test.go diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index dd36a84..e0601d8 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -17,6 +17,24 @@ type Redirect struct { StatusCode int } +// rewriteURL returns the destination URL and true if r matches reqURL. +func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) { + // check if from url matches request url + if strings.TrimSuffix(r.From, "/") == strings.TrimSuffix(reqURL, "/") { + return r.To, true + } + // handle wildcard redirects + trimmedFromURL := strings.TrimSuffix(r.From, "/*") + if strings.HasSuffix(r.From, "/*") && strings.HasPrefix(reqURL, trimmedFromURL) { + if strings.Contains(r.To, ":splat") { + splatURL := strings.ReplaceAll(r.To, ":splat", strings.TrimPrefix(reqURL, trimmedFromURL)) + return splatURL, true + } + return r.To, true + } + return "", false +} + // redirectsCacheTimeout specifies the timeout for the redirects cache. var redirectsCacheTimeout = 10 * time.Minute @@ -64,52 +82,21 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.I } func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) { - if len(redirects) > 0 { - for _, redirect := range redirects { - reqUrl := ctx.Req.RequestURI - // remove repo and branch from request url - reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo) - reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch) + reqURL := ctx.Req.RequestURI + // remove repo and branch from request url + reqURL = strings.TrimPrefix(reqURL, "/"+o.TargetRepo) + reqURL = strings.TrimPrefix(reqURL, "/@"+o.TargetBranch) - // check if from url matches request url - if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") { - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = redirect.To - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(redirect.To, redirect.StatusCode) - return true - } - } - - // handle wildcard redirects - trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*") - if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) { - if strings.Contains(redirect.To, ":splat") { - splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl)) - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = splatUrl - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(splatUrl, redirect.StatusCode) - return true - } - } else { - // do rewrite if status code is 200 - if redirect.StatusCode == 200 { - o.TargetPath = redirect.To - o.Upstream(ctx, giteaClient, redirectsCache) - return true - } else { - ctx.Redirect(redirect.To, redirect.StatusCode) - return true - } - } + for _, redirect := range redirects { + if dstURL, ok := redirect.rewriteURL(reqURL); ok { + // do rewrite if status code is 200 + if redirect.StatusCode == 200 { + o.TargetPath = dstURL + o.Upstream(ctx, giteaClient, redirectsCache) + } else { + ctx.Redirect(dstURL, redirect.StatusCode) } + return true } } diff --git a/server/upstream/redirects_test.go b/server/upstream/redirects_test.go new file mode 100644 index 0000000..c1088a9 --- /dev/null +++ b/server/upstream/redirects_test.go @@ -0,0 +1,37 @@ +package upstream + +import ( + "testing" +) + +func TestRedirect_rewriteURL(t *testing.T) { + for _, tc := range []struct { + redirect Redirect + reqURL string + wantDstURL string + wantOk bool + }{ + {Redirect{"/", "/dst", 200}, "/", "/dst", true}, + {Redirect{"/", "/dst", 200}, "/foo", "", false}, + {Redirect{"/src", "/dst", 200}, "/src", "/dst", true}, + {Redirect{"/src", "/dst", 200}, "/foo", "", false}, + {Redirect{"/src", "/dst", 200}, "/src/foo", "", false}, + {Redirect{"/*", "/dst", 200}, "/", "/dst", true}, + {Redirect{"/*", "/dst", 200}, "/src", "/dst", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true}, + // TODO: There shouldn't be double-slashes in these URLs: + // https://codeberg.org/Codeberg/pages-server/issues/269 + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst//", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst//foo", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst//foo/bar", true}, + // TODO: This is a workaround for the above bug. Should it be preserved? + {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dst/foo", true}, + // TODO: This behavior is incorrect; no redirect should be performed. + {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "/dst", true}, + } { + if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk { + t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v", + tc.redirect, tc.reqURL, dstURL, ok, tc.wantDstURL, tc.wantOk) + } + } +} From a986a52755aba425b9a69c94ed48ceb01c56776a Mon Sep 17 00:00:00 2001 From: Moritz Marquardt Date: Thu, 18 Apr 2024 21:19:45 +0000 Subject: [PATCH 7/8] Fix masked error message from Gitea (#306) This would yield to the error "forge client failed" instead of e.g. "404 Not Found". The issue was introduced in cbb2ce6d0732bcf2372cbc201fc1c1f2733aadba. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/306 Reviewed-by: crapStone Co-authored-by: Moritz Marquardt Co-committed-by: Moritz Marquardt --- server/handler/try.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/handler/try.go b/server/handler/try.go index 145b1a9..e76891d 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -1,6 +1,7 @@ package handler import ( + "fmt" "net/http" "strings" @@ -41,7 +42,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, // Try to request the file from the Gitea API if !options.Upstream(ctx, giteaClient, redirectsCache) { - html.ReturnErrorPage(ctx, "forge client failed", ctx.StatusCode) + html.ReturnErrorPage(ctx, fmt.Sprintf("Forge returned %d %s", ctx.StatusCode, http.StatusText(ctx.StatusCode)), ctx.StatusCode) } } From 69fb22a9e793e92e23e74900950ac9f60a049f9d Mon Sep 17 00:00:00 2001 From: Daniel Erat Date: Sat, 20 Apr 2024 11:00:15 +0000 Subject: [PATCH 8/8] Avoid extra slashes in redirects with :splat (#308) Remove leading slashes from captured portions of paths when redirecting using splats. This makes a directive like "/articles/* /posts/:splat 302" behave as described in FEATURES.md, i.e. "/articles/foo" now redirects to "/posts/foo" rather than to "/posts//foo". Fixes #269. This also changes the behavior of a redirect like "/articles/* /posts:splat 302". "/articles/foo" will now redirect to "/postsfoo" rather than to "/posts/foo". This change also fixes an issue where paths like "/articles123" would be incorrectly matched by the above patterns. Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/308 Reviewed-by: crapStone Co-authored-by: Daniel Erat Co-committed-by: Daniel Erat --- server/upstream/redirects.go | 15 +++++++++------ server/upstream/redirects_test.go | 17 ++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index e0601d8..ddde375 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -24,13 +24,16 @@ func (r *Redirect) rewriteURL(reqURL string) (dstURL string, ok bool) { return r.To, true } // handle wildcard redirects - trimmedFromURL := strings.TrimSuffix(r.From, "/*") - if strings.HasSuffix(r.From, "/*") && strings.HasPrefix(reqURL, trimmedFromURL) { - if strings.Contains(r.To, ":splat") { - splatURL := strings.ReplaceAll(r.To, ":splat", strings.TrimPrefix(reqURL, trimmedFromURL)) - return splatURL, true + if strings.HasSuffix(r.From, "/*") { + trimmedFromURL := strings.TrimSuffix(r.From, "/*") + if reqURL == trimmedFromURL || strings.HasPrefix(reqURL, trimmedFromURL+"/") { + if strings.Contains(r.To, ":splat") { + matched := strings.TrimPrefix(reqURL, trimmedFromURL) + matched = strings.TrimPrefix(matched, "/") + return strings.ReplaceAll(r.To, ":splat", matched), true + } + return r.To, true } - return r.To, true } return "", false } diff --git a/server/upstream/redirects_test.go b/server/upstream/redirects_test.go index c1088a9..6118a70 100644 --- a/server/upstream/redirects_test.go +++ b/server/upstream/redirects_test.go @@ -19,15 +19,14 @@ func TestRedirect_rewriteURL(t *testing.T) { {Redirect{"/*", "/dst", 200}, "/", "/dst", true}, {Redirect{"/*", "/dst", 200}, "/src", "/dst", true}, {Redirect{"/src/*", "/dst/:splat", 200}, "/src", "/dst/", true}, - // TODO: There shouldn't be double-slashes in these URLs: - // https://codeberg.org/Codeberg/pages-server/issues/269 - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst//", true}, - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst//foo", true}, - {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst//foo/bar", true}, - // TODO: This is a workaround for the above bug. Should it be preserved? - {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dst/foo", true}, - // TODO: This behavior is incorrect; no redirect should be performed. - {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "/dst", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/", "/dst/", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo", "/dst/foo", true}, + {Redirect{"/src/*", "/dst/:splat", 200}, "/src/foo/bar", "/dst/foo/bar", true}, + {Redirect{"/src/*", "/dst/:splatsuffix", 200}, "/src/foo", "/dst/foosuffix", true}, + {Redirect{"/src/*", "/dst:splat", 200}, "/src/foo", "/dstfoo", true}, + {Redirect{"/src/*", "/dst", 200}, "/srcfoo", "", false}, + // This is the example from FEATURES.md: + {Redirect{"/articles/*", "/posts/:splat", 302}, "/articles/2022/10/12/post-1/", "/posts/2022/10/12/post-1/", true}, } { if dstURL, ok := tc.redirect.rewriteURL(tc.reqURL); dstURL != tc.wantDstURL || ok != tc.wantOk { t.Errorf("%#v.rewriteURL(%q) = %q, %v; want %q, %v",