From 16a8d5d5750d5f59ab3fb26c47c13d38f53480e6 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 28 Aug 2022 20:54:17 +0200 Subject: [PATCH] REMOVE fasthttp version --- cmd/main.go | 2 - cmd/main_fasthttp.go | 154 -------------- go.mod | 2 +- html/error.go | 16 ++ html/error_fasthttp.go | 17 -- html/error_std.go | 23 -- integration/get_test.go | 2 +- server/dns/const.go | 6 - server/dns/dns.go | 4 + server/gitea/client.go | 162 ++++++++++++++ server/gitea/client_fasthttp.go | 174 --------------- server/gitea/client_fasthttp_test.go | 25 --- server/gitea/client_std.go | 168 --------------- server/handler.go | 2 - server/handler_fasthttp.go | 307 --------------------------- server/handler_fasthttp_test.go | 51 ----- server/handler_test.go | 2 - server/setup.go | 2 - server/setup_fasthttp.go | 46 ---- server/try.go | 2 - server/try_fasthttp.go | 50 ----- server/upstream/const.go | 8 - server/upstream/domains.go | 6 + server/upstream/upstream.go | 141 ++++++++++++ server/upstream/upstream_fasthttp.go | 150 ------------- server/upstream/upstream_std.go | 148 ------------- 26 files changed, 331 insertions(+), 1339 deletions(-) delete mode 100644 cmd/main_fasthttp.go delete mode 100644 html/error_fasthttp.go delete mode 100644 html/error_std.go delete mode 100644 server/dns/const.go delete mode 100644 server/gitea/client_fasthttp.go delete mode 100644 server/gitea/client_fasthttp_test.go delete mode 100644 server/gitea/client_std.go delete mode 100644 server/handler_fasthttp.go delete mode 100644 server/handler_fasthttp_test.go delete mode 100644 server/setup_fasthttp.go delete mode 100644 server/try_fasthttp.go delete mode 100644 server/upstream/const.go delete mode 100644 server/upstream/upstream_fasthttp.go delete mode 100644 server/upstream/upstream_std.go diff --git a/cmd/main.go b/cmd/main.go index bd9e4ad..5e0ce6f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,5 +1,3 @@ -//go:build !fasthttp - package cmd import ( diff --git a/cmd/main_fasthttp.go b/cmd/main_fasthttp.go deleted file mode 100644 index c15bcb8..0000000 --- a/cmd/main_fasthttp.go +++ /dev/null @@ -1,154 +0,0 @@ -//go:build fasthttp - -package cmd - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "net" - "os" - "strings" - "time" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/database" - "codeberg.org/codeberg/pages/server/gitea" -) - -// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. -// TODO: make it a flag -var AllowedCorsDomains = []string{ - "fonts.codeberg.org", - "design.codeberg.org", -} - -// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. -// TODO: Make it a flag too -var BlacklistedPaths = []string{ - "/.well-known/acme-challenge/", -} - -// Serve sets up and starts the web server. -func Serve(ctx *cli.Context) error { - // Initalize the logger. - logLevel, err := zerolog.ParseLevel(ctx.String("log-level")) - if err != nil { - return err - } - log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) - - giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") - giteaAPIToken := ctx.String("gitea-api-token") - rawDomain := ctx.String("raw-domain") - mainDomainSuffix := ctx.String("pages-domain") - rawInfoPage := ctx.String("raw-info-page") - listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) - enableHTTPServer := ctx.Bool("enable-http-server") - - acmeAPI := ctx.String("acme-api-endpoint") - acmeMail := ctx.String("acme-email") - acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") - acmeAcceptTerms := ctx.Bool("acme-accept-terms") - acmeEabKID := ctx.String("acme-eab-kid") - acmeEabHmac := ctx.String("acme-eab-hmac") - dnsProvider := ctx.String("dns-provider") - if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { - return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory") - } - - allowedCorsDomains := AllowedCorsDomains - if len(rawDomain) != 0 { - allowedCorsDomains = append(allowedCorsDomains, rawDomain) - } - - // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash - if !strings.HasPrefix(mainDomainSuffix, ".") { - mainDomainSuffix = "." + mainDomainSuffix - } - - keyCache := cache.NewKeyValueCache() - challengeCache := cache.NewKeyValueCache() - // canonicalDomainCache stores canonical domains - canonicalDomainCache := cache.NewKeyValueCache() - // dnsLookupCache stores DNS lookups for custom domains - dnsLookupCache := cache.NewKeyValueCache() - // clientResponseCache stores responses from the Gitea server - clientResponseCache := cache.NewKeyValueCache() - - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) - if err != nil { - return fmt.Errorf("could not create new gitea client: %v", err) - } - - // Create handler based on settings - handler := server.Handler(mainDomainSuffix, rawDomain, - giteaClient, - giteaRoot, rawInfoPage, - BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache) - - fastServer := server.SetupServer(handler) - httpServer := server.SetupHTTPACMEChallengeServer(challengeCache) - - // Setup listener and TLS - log.Info().Msgf("Listening on https://%s", listeningAddress) - listener, err := net.Listen("tcp", listeningAddress) - if err != nil { - return fmt.Errorf("couldn't create listener: %v", err) - } - - // TODO: make "key-database.pogreb" set via flag - certDB, err := database.New("key-database.pogreb") - if err != nil { - return fmt.Errorf("could not create database: %v", err) - } - defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it - - listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, - giteaClient, - dnsProvider, - acmeUseRateLimits, - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, - certDB)) - - acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) - if err != nil { - return err - } - - if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil { - return err - } - - interval := 12 * time.Hour - certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) - defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB) - - if enableHTTPServer { - go func() { - log.Info().Msg("Start HTTP server listening on :80") - err := httpServer.ListenAndServe("[::]:80") - if err != nil { - log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") - } - }() - } - - // Start the web fastServer - log.Info().Msgf("Start listening on %s", listener.Addr()) - err = fastServer.Serve(listener) - if err != nil { - log.Panic().Err(err).Msg("Couldn't start fastServer") - } - - return nil -} diff --git a/go.mod b/go.mod index 58e0c4d..560e488 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module codeberg.org/codeberg/pages -go 1.18 +go 1.19 require ( code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa diff --git a/html/error.go b/html/error.go index b1ccc06..a0d0ae6 100644 --- a/html/error.go +++ b/html/error.go @@ -1,11 +1,27 @@ package html import ( + "io" "net/http" "strconv" "strings" + + "codeberg.org/codeberg/pages/server/context" ) +// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced +// with the provided status code. +func ReturnErrorPage(ctx *context.Context, msg string, code int) { + ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") + ctx.RespWriter.WriteHeader(code) + + if msg == "" { + msg = errorBody(code) + } + + _, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg)) +} + func errorMessage(statusCode int) string { message := http.StatusText(statusCode) diff --git a/html/error_fasthttp.go b/html/error_fasthttp.go deleted file mode 100644 index 2180224..0000000 --- a/html/error_fasthttp.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build fasthttp - -package html - -import ( - "github.com/valyala/fasthttp" -) - -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) { - ctx.Response.SetStatusCode(code) - ctx.Response.Header.SetContentType("text/html; charset=utf-8") - - // TODO: use template engine? - ctx.Response.SetBody([]byte(errorBody(code))) -} diff --git a/html/error_std.go b/html/error_std.go deleted file mode 100644 index 4d745d4..0000000 --- a/html/error_std.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build !fasthttp - -package html - -import ( - "io" - "strings" - - "codeberg.org/codeberg/pages/server/context" -) - -// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced -// with the provided status code. -func ReturnErrorPage(ctx *context.Context, msg string, code int) { - ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8") - ctx.RespWriter.WriteHeader(code) - - if msg == "" { - msg = errorBody(code) - } - - _, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg)) -} diff --git a/integration/get_test.go b/integration/get_test.go index 6054e17..7e9fcca 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -25,7 +25,7 @@ func TestGetRedirect(t *testing.T) { t.FailNow() } assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location")) - assert.EqualValues(t, 0, getSize(resp.Body)) + assert.EqualValues(t, `Temporary Redirect.`, strings.TrimSpace(string(getBytes(resp.Body)))) } func TestGetContent(t *testing.T) { diff --git a/server/dns/const.go b/server/dns/const.go deleted file mode 100644 index bb2413b..0000000 --- a/server/dns/const.go +++ /dev/null @@ -1,6 +0,0 @@ -package dns - -import "time" - -// lookupCacheTimeout specifies the timeout for the DNS lookup cache. -var lookupCacheTimeout = 15 * time.Minute diff --git a/server/dns/dns.go b/server/dns/dns.go index dc759b0..818e29a 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -3,10 +3,14 @@ package dns import ( "net" "strings" + "time" "codeberg.org/codeberg/pages/server/cache" ) +// lookupCacheTimeout specifies the timeout for the DNS lookup cache. +var lookupCacheTimeout = 15 * time.Minute + // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { diff --git a/server/gitea/client.go b/server/gitea/client.go index 3c1bb49..ee2f8b3 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -2,6 +2,17 @@ package gitea import ( "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "code.gitea.io/sdk/gitea" + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/server/cache" ) var ErrorNotFound = errors.New("not found") @@ -11,3 +22,154 @@ const ( defaultBranchCacheKeyPrefix = "defaultBranch" giteaObjectTypeHeader = "X-Gitea-Object-Type" ) + +type Client struct { + sdkClient *gitea.Client + responseCache cache.SetGetKey + + followSymlinks bool + supportLFS bool +} + +func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { + rootURL, err := url.Parse(giteaRoot) + if err != nil { + return nil, err + } + giteaRoot = strings.Trim(rootURL.String(), "/") + + stdClient := http.Client{Timeout: 10 * time.Second} + + sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) + return &Client{ + sdkClient: sdk, + responseCache: respCache, + followSymlinks: followSymlinks, + supportLFS: supportLFS, + }, err +} + +func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { + reader, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) + if err != nil { + return nil, err + } + defer reader.Close() + return io.ReadAll(reader) +} + +func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, *http.Response, error) { + // if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { + // cachedResponse = cachedValue.(gitea.FileResponse) + reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) + if resp != nil { + switch resp.StatusCode { + case http.StatusOK: + + // add caching + + // Write the response body to the original request + // var cacheBodyWriter bytes.Buffer + // if res != nil { + // if res.Header.ContentLength() > fileCacheSizeLimit { + // // fasthttp else will set "Content-Length: 0" + // ctx.Response().SetBodyStream(&strings.Reader{}, -1) + // + // err = res.BodyWriteTo(ctx.Response.BodyWriter()) + // } else { + // // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? + // err = res.BodyWriteTo(io.MultiWriter(ctx.Response().BodyWriter(), &cacheBodyWriter)) + // } + // } else { + // _, err = ctx.Write(cachedResponse.Body) + // } + + // if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil { + // cachedResponse.Exists = true + // cachedResponse.MimeType = mimeType + // cachedResponse.Body = cacheBodyWriter.Bytes() + // _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) + // } + // store ETag in resp !!!! + + objType := resp.Header.Get(giteaObjectTypeHeader) + log.Trace().Msgf("server raw content object: %s", objType) + if client.followSymlinks && objType == "symlink" { + // limit to 1000 chars + defer reader.Close() + linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000)) + if err != nil { + return nil, nil, err + } + linkDest := strings.TrimSpace(string(linkDestBytes)) + + log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } + + return reader, resp.Response, err + case http.StatusNotFound: + + // add not exist caching + // _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ + // Exists: false, + // }, fileCacheTimeout) + + return nil, resp.Response, ErrorNotFound + default: + return nil, resp.Response, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) + } + } + return nil, nil, err +} + +func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) { + cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) + + if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { + return stamp.(*BranchTimestamp), nil + } + + branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return &BranchTimestamp{}, ErrorNotFound + } + return &BranchTimestamp{}, err + } + if resp.StatusCode != http.StatusOK { + return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) + } + + stamp := &BranchTimestamp{ + Branch: branch.Name, + Timestamp: branch.Commit.Timestamp, + } + + if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { + log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) + } + return stamp, nil +} + +func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { + cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) + + if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil { + return branch.(string), nil + } + + repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode) + } + + branch := repo.DefaultBranch + if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { + log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName) + } + return branch, nil +} diff --git a/server/gitea/client_fasthttp.go b/server/gitea/client_fasthttp.go deleted file mode 100644 index 904272b..0000000 --- a/server/gitea/client_fasthttp.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build fasthttp - -package gitea - -import ( - "fmt" - "net/url" - "strings" - "time" - - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - "github.com/valyala/fastjson" - - "codeberg.org/codeberg/pages/server/cache" -) - -const ( - giteaAPIRepos = "/api/v1/repos/" -) - -type Client struct { - giteaRoot string - giteaAPIToken string - infoTimeout time.Duration - contentTimeout time.Duration - fastClient *fasthttp.Client - responseCache cache.SetGetKey - - followSymlinks bool - supportLFS bool -} - -func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { - rootURL, err := url.Parse(giteaRoot) - giteaRoot = strings.Trim(rootURL.String(), "/") - - return &Client{ - giteaRoot: giteaRoot, - giteaAPIToken: giteaAPIToken, - infoTimeout: 5 * time.Second, - contentTimeout: 10 * time.Second, - fastClient: getFastHTTPClient(), - responseCache: respCache, - - followSymlinks: followSymlinks, - supportLFS: supportLFS, - }, err -} - -func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { - resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) - if err != nil { - return nil, err - } - return resp.Body(), nil -} - -func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { - var apiURL string - if client.supportLFS { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref)) - } else { - apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - } - resp, err := client.do(client.contentTimeout, apiURL) - if err != nil { - return nil, err - } - - switch resp.StatusCode() { - case fasthttp.StatusOK: - objType := string(resp.Header.Peek(giteaObjectTypeHeader)) - log.Trace().Msgf("server raw content object: %s", objType) - if client.followSymlinks && objType == "symlink" { - // TODO: limit to 1000 chars if we switched to std - linkDest := strings.TrimSpace(string(resp.Body())) - log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) - return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) - } - - return resp, nil - - case fasthttp.StatusNotFound: - return nil, ErrorNotFound - - default: - return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode()) - } -} - -func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) { - cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) - - if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { - return stamp.(*BranchTimestamp), nil - } - - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName) - res, err := client.do(client.infoTimeout, url) - if err != nil { - return &BranchTimestamp{}, err - } - if res.StatusCode() != fasthttp.StatusOK { - return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - timestamp, err := time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp")) - if err != nil { - return &BranchTimestamp{}, err - } - - stamp := &BranchTimestamp{ - Branch: branchName, - Timestamp: timestamp, - } - - client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout) - return stamp, nil -} - -func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { - cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) - - if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil { - return branch.(string), nil - } - - url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName) - res, err := client.do(client.infoTimeout, url) - if err != nil { - return "", err - } - if res.StatusCode() != fasthttp.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } - - branch := fastjson.GetString(res.Body(), "default_branch") - client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout) - return branch, nil -} - -func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) { - req := fasthttp.AcquireRequest() - - req.SetRequestURI(url) - req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken) - res := fasthttp.AcquireResponse() - - err := client.fastClient.DoTimeout(req, res, timeout) - - return res, err -} - -// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package -func joinURL(baseURL string, paths ...string) string { - p := make([]string, 0, len(paths)) - for i := range paths { - path := strings.TrimSpace(paths[i]) - path = strings.Trim(path, "/") - if len(path) != 0 { - p = append(p, path) - } - } - - return baseURL + "/" + strings.Join(p, "/") -} - -func getFastHTTPClient() *fasthttp.Client { - return &fasthttp.Client{ - MaxConnDuration: 60 * time.Second, - MaxConnWaitTimeout: 1000 * time.Millisecond, - MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea! - } -} diff --git a/server/gitea/client_fasthttp_test.go b/server/gitea/client_fasthttp_test.go deleted file mode 100644 index f92a450..0000000 --- a/server/gitea/client_fasthttp_test.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build fasthttp - -package gitea - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestJoinURL(t *testing.T) { - baseURL := "" - assert.EqualValues(t, "/", joinURL(baseURL)) - assert.EqualValues(t, "/", joinURL(baseURL, "", "")) - - baseURL = "http://wwow.url.com" - assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d")) - - baseURL = "http://wow.url.com/subpath/2" - assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg")) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main"))) - assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main")) -} diff --git a/server/gitea/client_std.go b/server/gitea/client_std.go deleted file mode 100644 index 67722b5..0000000 --- a/server/gitea/client_std.go +++ /dev/null @@ -1,168 +0,0 @@ -//go:build !fasthttp - -package gitea - -import ( - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "code.gitea.io/sdk/gitea" - "github.com/rs/zerolog/log" - - "codeberg.org/codeberg/pages/server/cache" -) - -type Client struct { - sdkClient *gitea.Client - responseCache cache.SetGetKey - - followSymlinks bool - supportLFS bool -} - -func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { - rootURL, err := url.Parse(giteaRoot) - if err != nil { - return nil, err - } - giteaRoot = strings.Trim(rootURL.String(), "/") - - stdClient := http.Client{Timeout: 10 * time.Second} - - sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken)) - return &Client{ - sdkClient: sdk, - responseCache: respCache, - followSymlinks: followSymlinks, - supportLFS: supportLFS, - }, err -} - -func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { - reader, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) - if err != nil { - return nil, err - } - defer reader.Close() - return io.ReadAll(reader) -} - -func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, *http.Response, error) { - // if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { - // cachedResponse = cachedValue.(gitea.FileResponse) - reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS) - if resp != nil { - switch resp.StatusCode { - case http.StatusOK: - - // add caching - - // Write the response body to the original request - // var cacheBodyWriter bytes.Buffer - // if res != nil { - // if res.Header.ContentLength() > fileCacheSizeLimit { - // // fasthttp else will set "Content-Length: 0" - // ctx.Response().SetBodyStream(&strings.Reader{}, -1) - // - // err = res.BodyWriteTo(ctx.Response.BodyWriter()) - // } else { - // // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick? - // err = res.BodyWriteTo(io.MultiWriter(ctx.Response().BodyWriter(), &cacheBodyWriter)) - // } - // } else { - // _, err = ctx.Write(cachedResponse.Body) - // } - - // if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil { - // cachedResponse.Exists = true - // cachedResponse.MimeType = mimeType - // cachedResponse.Body = cacheBodyWriter.Bytes() - // _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout) - // } - // store ETag in resp !!!! - - objType := resp.Header.Get(giteaObjectTypeHeader) - log.Trace().Msgf("server raw content object: %s", objType) - if client.followSymlinks && objType == "symlink" { - // limit to 1000 chars - defer reader.Close() - linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000)) - if err != nil { - return nil, nil, err - } - linkDest := strings.TrimSpace(string(linkDestBytes)) - - log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) - return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) - } - - return reader, resp.Response, err - case http.StatusNotFound: - - // add not exist caching - // _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{ - // Exists: false, - // }, fileCacheTimeout) - - return nil, resp.Response, ErrorNotFound - default: - return nil, resp.Response, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) - } - } - return nil, nil, err -} - -func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) { - cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName) - - if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil { - return stamp.(*BranchTimestamp), nil - } - - branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName) - if err != nil { - if resp != nil && resp.StatusCode == http.StatusNotFound { - return &BranchTimestamp{}, ErrorNotFound - } - return &BranchTimestamp{}, err - } - if resp.StatusCode != http.StatusOK { - return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode) - } - - stamp := &BranchTimestamp{ - Branch: branch.Name, - Timestamp: branch.Commit.Timestamp, - } - - if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil { - log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName) - } - return stamp, nil -} - -func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) { - cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName) - - if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil { - return branch.(string), nil - } - - repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode) - } - - branch := repo.DefaultBranch - if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil { - log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName) - } - return branch, nil -} diff --git a/server/handler.go b/server/handler.go index 7d48013..c9ade2c 100644 --- a/server/handler.go +++ b/server/handler.go @@ -1,5 +1,3 @@ -//go:build !fasthttp - package server import ( diff --git a/server/handler_fasthttp.go b/server/handler_fasthttp.go deleted file mode 100644 index 628455a..0000000 --- a/server/handler_fasthttp.go +++ /dev/null @@ -1,307 +0,0 @@ -//go:build fasthttp - -package server - -import ( - "bytes" - "strings" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/dns" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/upstream" - "codeberg.org/codeberg/pages/server/utils" - "codeberg.org/codeberg/pages/server/version" -) - -// Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain string, - giteaClient *gitea.Client, - giteaRoot, rawInfoPage string, - blacklistedPaths, allowedCorsDomains []string, - dnsLookupCache, canonicalDomainCache cache.SetGetKey, -) func(ctx *fasthttp.RequestCtx) { - return func(ctx *fasthttp.RequestCtx) { - log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger() - - ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version) - - // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin - ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin") - - // Enable browser caching for up to 10 minutes - ctx.Response.Header.Set("Cache-Control", "public, max-age=600") - - trimmedHost := utils.TrimHostPort(string(ctx.Request.Host())) - - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { - ctx.Response.Header.Set("Strict-Transport-Security", hsts) - } - - // Block all methods not required for static pages - if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() { - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed) - return - } - - // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range blacklistedPaths { - if strings.HasPrefix(string(ctx.Path()), blacklistedPath) { - html.ReturnErrorPage(ctx, fasthttp.StatusForbidden) - return - } - } - - // Allow CORS for specified domains - allowCors := false - for _, allowedCorsDomain := range allowedCorsDomains { - if strings.EqualFold(trimmedHost, allowedCorsDomain) { - allowCors = true - break - } - } - if allowCors { - ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") - ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD") - } - ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS") - if ctx.IsOptions() { - ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent) - return - } - - // Prepare request information to Gitea - var targetOwner, targetRepo, targetBranch, targetPath string - targetOptions := &upstream.Options{ - TryIndexPages: true, - } - - // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will - // also disallow search indexing and add a Link header to the canonical URL. - tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool { - if repo == "" { - log.Debug().Msg("tryBranch: repo == ''") - return false - } - - // Replace "~" to "/" so we can access branch that contains slash character - // Branch name cannot contain "~" so doing this is okay - branch = strings.ReplaceAll(branch, "~", "/") - - // Check if the branch exists, otherwise treat it as a file path - branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch) - if branchTimestampResult == nil { - log.Debug().Msg("tryBranch: branch doesn't exist") - return false - } - - // Branch exists, use it - targetRepo = repo - targetPath = strings.Trim(strings.Join(path, "/"), "/") - targetBranch = branchTimestampResult.Branch - - targetOptions.BranchTimestamp = branchTimestampResult.Timestamp - - if canonicalLink != "" { - // Hide from search machines & add canonical link - ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex") - ctx.Response.Header.Set("Link", - strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+ - "; rel=\"canonical\"", - ) - } - - log.Debug().Msg("tryBranch: true") - return true - } - - log.Debug().Msg("preparations") - if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { - // Serve raw content from RawDomain - log.Debug().Msg("raw domain") - - targetOptions.TryIndexPages = false - if targetOptions.ForbiddenMimeTypes == nil { - targetOptions.ForbiddenMimeTypes = make(map[string]bool) - } - targetOptions.ForbiddenMimeTypes["text/html"] = true - targetOptions.DefaultMimeType = "text/plain; charset=utf-8" - - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - if len(pathElements) < 2 { - // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required - ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect) - return - } - targetOwner = pathElements[0] - targetRepo = pathElements[1] - - // raw.codeberg.org/example/myrepo/@main/index.html - if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") { - log.Debug().Msg("raw domain preparations, now trying with specified branch") - if tryBranch(log, - targetRepo, pathElements[2][1:], pathElements[3:], - giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream 1") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - log.Debug().Msg("missing branch") - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - - log.Debug().Msg("raw domain preparations, now trying with default branch") - tryBranch(log, - targetRepo, "", pathElements[2:], - giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p", - ) - log.Debug().Msg("tryBranch, now trying upstream 2") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - - } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - // Serve pages from subdomains of MainDomainSuffix - log.Debug().Msg("main domain suffix") - - pathElements := strings.Split(strings.Trim(string(ctx.Request.URI().Path()), "/"), "/") - targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix) - targetRepo = pathElements[0] - targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/") - - if targetOwner == "www" { - // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname? - ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect) - return - } - - // Check if the first directory is a repo with the second directory as a branch - // example.codeberg.page/myrepo/@main/index.html - if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") { - if targetRepo == "pages" { - // example.codeberg.org/pages/@... redirects to example.codeberg.org/@... - ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect) - return - } - - log.Debug().Msg("main domain preparations, now trying with specified repo & branch") - if tryBranch(log, - pathElements[0], pathElements[1][1:], pathElements[2:], - "/"+pathElements[0]+"/%p", - ) { - log.Debug().Msg("tryBranch, now trying upstream 3") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - } else { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - } - return - } - - // Check if the first directory is a branch for the "pages" repo - // example.codeberg.page/@main/index.html - if strings.HasPrefix(pathElements[0], "@") { - log.Debug().Msg("main domain preparations, now trying with specified branch") - if tryBranch(log, - "pages", pathElements[0][1:], pathElements[1:], "/%p") { - log.Debug().Msg("tryBranch, now trying upstream 4") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - } else { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - } - return - } - - // Check if the first directory is a repo with a "pages" branch - // example.codeberg.page/myrepo/index.html - // example.codeberg.page/pages/... is not allowed here. - log.Debug().Msg("main domain preparations, now trying with specified repo") - if pathElements[0] != "pages" && tryBranch(log, - pathElements[0], "pages", pathElements[1:], "") { - log.Debug().Msg("tryBranch, now trying upstream 5") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - // Try to use the "pages" repo on its default branch - // example.codeberg.page/index.html - log.Debug().Msg("main domain preparations, now trying with default repo/branch") - if tryBranch(log, - "pages", "", pathElements, "") { - log.Debug().Msg("tryBranch, now trying upstream 6") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - // Couldn't find a valid repo/branch - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } else { - trimmedHostStr := string(trimmedHost) - - // Serve pages from external domains - targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache) - if targetOwner == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - - pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/") - canonicalLink := "" - if strings.HasPrefix(pathElements[0], "@") { - targetBranch = pathElements[0][1:] - pathElements = pathElements[1:] - canonicalLink = "/%p" - } - - // Try to use the given repo on the given branch or the default branch - log.Debug().Msg("custom domain preparations, now trying with details from DNS") - if tryBranch(log, - targetRepo, targetBranch, pathElements, canonicalLink) { - canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache) - if !valid { - html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest) - return - } else if canonicalDomain != trimmedHostStr { - // only redirect if the target is also a codeberg page! - targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache) - if targetOwner != "" { - ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect) - return - } - - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - - log.Debug().Msg("tryBranch, now trying upstream 7") - tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, - targetOptions, targetOwner, targetRepo, targetBranch, targetPath, - canonicalDomainCache) - return - } - - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return - } - } -} diff --git a/server/handler_fasthttp_test.go b/server/handler_fasthttp_test.go deleted file mode 100644 index 35fedbe..0000000 --- a/server/handler_fasthttp_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build fasthttp - -package server - -import ( - "fmt" - "testing" - "time" - - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/gitea" -) - -func TestHandlerPerformance(t *testing.T) { - giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false) - testHandler := Handler( - "codeberg.page", "raw.codeberg.org", - giteaClient, - giteaRoot, "https://docs.codeberg.org/pages/raw-content/", - []string{"/.well-known/acme-challenge/"}, - []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - ) - - testCase := func(uri string, status int) { - ctx := &fasthttp.RequestCtx{ - Request: *fasthttp.AcquireRequest(), - Response: *fasthttp.AcquireResponse(), - } - ctx.Request.SetRequestURI(uri) - fmt.Printf("Start: %v\n", time.Now()) - start := time.Now() - testHandler(ctx) - end := time.Now() - fmt.Printf("Done: %v\n", time.Now()) - if ctx.Response.StatusCode() != status { - t.Errorf("request failed with status code %d", ctx.Response.StatusCode()) - } else { - t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds()) - } - } - - testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200 - testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200 - testCase("https://example.momar.xyz/", 424) // TODO: expect 200 - testCase("https://codeberg.page/", 424) // TODO: expect 200 -} diff --git a/server/handler_test.go b/server/handler_test.go index 4bfbad7..c0aca14 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1,5 +1,3 @@ -//go:build !fasthttp - package server import ( diff --git a/server/setup.go b/server/setup.go index b0110d0..33204c0 100644 --- a/server/setup.go +++ b/server/setup.go @@ -1,5 +1,3 @@ -//go:build !fasthttp - package server import ( diff --git a/server/setup_fasthttp.go b/server/setup_fasthttp.go deleted file mode 100644 index 98aab45..0000000 --- a/server/setup_fasthttp.go +++ /dev/null @@ -1,46 +0,0 @@ -//go:build fasthttp - -package server - -import ( - "net/http" - "strings" - "time" - - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/utils" -) - -func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server { - // Enable compression by wrapping the handler with the compression function provided by FastHTTP - compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed) - - return &fasthttp.Server{ - Handler: compressedHandler, - DisablePreParseMultipartForm: true, - NoDefaultServerHeader: true, - NoDefaultDate: true, - ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge - } -} - -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server { - challengePath := "/.well-known/acme-challenge/" - - return &fasthttp.Server{ - Handler: func(ctx *fasthttp.RequestCtx) { - if strings.HasPrefix(string(ctx.Path()), challengePath) { - challenge, ok := challengeCache.Get(utils.TrimHostPort(string(ctx.Host())) + "/" + strings.TrimPrefix(string(ctx.Path()), challengePath)) - if !ok || challenge == nil { - ctx.SetStatusCode(http.StatusNotFound) - ctx.SetBodyString("no challenge for this token") - } - ctx.SetBodyString(challenge.(string)) - } else { - ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently) - } - }, - } -} diff --git a/server/try.go b/server/try.go index 656446c..c209d29 100644 --- a/server/try.go +++ b/server/try.go @@ -1,5 +1,3 @@ -//go:build !fasthttp - package server import ( diff --git a/server/try_fasthttp.go b/server/try_fasthttp.go deleted file mode 100644 index c4913a1..0000000 --- a/server/try_fasthttp.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build fasthttp - -package server - -import ( - "strings" - - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/upstream" -) - -// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure. -func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, - mainDomainSuffix, trimmedHost string, - - targetOptions *upstream.Options, - targetOwner, targetRepo, targetBranch, targetPath string, - - canonicalDomainCache cache.SetGetKey, -) { - // check if a canonical domain exists on a request on MainDomain - if strings.HasSuffix(trimmedHost, mainDomainSuffix) { - canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache) - if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) { - canonicalPath := string(ctx.RequestURI()) - if targetRepo != "pages" { - path := strings.SplitN(canonicalPath, "/", 3) - if len(path) >= 3 { - canonicalPath = "/" + path[2] - } - } - ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect) - return - } - } - - targetOptions.TargetOwner = targetOwner - targetOptions.TargetRepo = targetRepo - targetOptions.TargetBranch = targetBranch - targetOptions.TargetPath = targetPath - - // Try to request the file from the Gitea API - if !targetOptions.Upstream(ctx, giteaClient) { - html.ReturnErrorPage(ctx, ctx.Response.StatusCode()) - } -} diff --git a/server/upstream/const.go b/server/upstream/const.go deleted file mode 100644 index 8a772d9..0000000 --- a/server/upstream/const.go +++ /dev/null @@ -1,8 +0,0 @@ -package upstream - -import "time" - -// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. -var canonicalDomainCacheTimeout = 15 * time.Minute - -const canonicalDomainConfig = ".domains" diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 553c148..95ba436 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -2,11 +2,17 @@ package upstream import ( "strings" + "time" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" ) +// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache. +var canonicalDomainCacheTimeout = 15 * time.Minute + +const canonicalDomainConfig = ".domains" + // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) { var ( diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 2afb5e6..09d2d63 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -1,7 +1,24 @@ package upstream import ( + "errors" + "io" + "net/http" + "strings" "time" + + "github.com/rs/zerolog/log" + + "codeberg.org/codeberg/pages/html" + "codeberg.org/codeberg/pages/server/context" + "codeberg.org/codeberg/pages/server/gitea" +) + +const ( + headerContentType = "Content-Type" + headerETag = "ETag" + headerLastModified = "Last-Modified" + headerIfModifiedSince = "If-Modified-Since" ) // upstreamIndexPages lists pages that may be considered as index pages for directories. @@ -32,3 +49,127 @@ type Options struct { appendTrailingSlash bool redirectIfExists string } + +// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) { + log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() + + // Check if the branch exists and when it was modified + if o.BranchTimestamp.IsZero() { + branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) + + if branch == nil { + html.ReturnErrorPage(ctx, "", http.StatusFailedDependency) + return true + } + o.TargetBranch = branch.Branch + o.BranchTimestamp = branch.Timestamp + } + + if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { + html.ReturnErrorPage(ctx, "", http.StatusBadRequest) + return true + } + + // Check if the browser has a cached version + if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil { + if !ifModifiedSince.Before(o.BranchTimestamp) { + ctx.RespWriter.WriteHeader(http.StatusNotModified) + return true + } + } + log.Debug().Msg("preparations") + + reader, res, err := giteaClient.ServeRawContent(o.generateUriClientArgs()) + log.Debug().Msg("acquisition") + + // Handle errors + if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil) { + if o.TryIndexPages { + // copy the o struct & try if an index page exists + optionsForIndexPages := *o + optionsForIndexPages.TryIndexPages = false + optionsForIndexPages.appendTrailingSlash = true + for _, indexPage := range upstreamIndexPages { + optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage + if optionsForIndexPages.Upstream(ctx, giteaClient) { + return true + } + } + // compatibility fix for GitHub Pages (/example → /example.html) + optionsForIndexPages.appendTrailingSlash = false + optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" + optionsForIndexPages.TargetPath = o.TargetPath + ".html" + if optionsForIndexPages.Upstream(ctx, giteaClient) { + return true + } + } + + ctx.Response().StatusCode = http.StatusNotFound + if o.TryIndexPages { + // copy the o struct & try if a not found page exists + optionsForNotFoundPages := *o + optionsForNotFoundPages.TryIndexPages = false + optionsForNotFoundPages.appendTrailingSlash = false + for _, notFoundPage := range upstreamNotFoundPages { + optionsForNotFoundPages.TargetPath = "/" + notFoundPage + if optionsForNotFoundPages.Upstream(ctx, giteaClient) { + return true + } + } + } + return false + } + if res != nil && (err != nil || res.StatusCode != http.StatusOK) { + log.Printf("Couldn't fetch contents (status code %d): %v\n", res.StatusCode, err) + html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) + return true + } + + // Append trailing slash if missing (for index files), and redirect to fix filenames in general + // o.appendTrailingSlash is only true when looking for index pages + if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { + ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) + return true + } + if strings.HasSuffix(ctx.Path(), "/index.html") { + ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) + return true + } + if o.redirectIfExists != "" { + ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect) + return true + } + log.Debug().Msg("error handling") + + // Set the MIME type + mimeType := o.getMimeTypeByExtension() + ctx.Response().Header.Set(headerContentType, mimeType) + + // Set ETag + if res != nil { + ctx.Response().Header.Set(headerETag, res.Header.Get(headerETag)) + } + + if ctx.Response().StatusCode != http.StatusNotFound { + // Everything's okay so far + ctx.Response().StatusCode = http.StatusOK + } + ctx.Response().Header.Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) + + log.Debug().Msg("response preparations") + + // Write the response body to the original request + if reader != nil { + _, err := io.Copy(ctx.RespWriter, reader) + if err != nil { + log.Printf("Couldn't write body: %s\n", err) + html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) + return true + } + } + + log.Debug().Msg("response") + + return true +} diff --git a/server/upstream/upstream_fasthttp.go b/server/upstream/upstream_fasthttp.go deleted file mode 100644 index b27b457..0000000 --- a/server/upstream/upstream_fasthttp.go +++ /dev/null @@ -1,150 +0,0 @@ -//go:build fasthttp - -package upstream - -import ( - "bytes" - "errors" - "fmt" - "strings" - "time" - - "github.com/rs/zerolog/log" - "github.com/valyala/fasthttp" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/gitea" -) - -// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client) (final bool) { - log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() - - // Check if the branch exists and when it was modified - if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) - - if branch == nil { - html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency) - return true - } - o.TargetBranch = branch.Branch - o.BranchTimestamp = branch.Timestamp - } - - if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { - html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest) - return true - } - - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil { - if !ifModifiedSince.Before(o.BranchTimestamp) { - ctx.Response.SetStatusCode(fasthttp.StatusNotModified) - return true - } - } - log.Debug().Msg("preparations") - - // Make a GET request to the upstream URL - uri := o.generateUri() - var cachedResponse gitea.FileResponse - // if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { - // cachedResponse = cachedValue.(gitea.FileResponse) - // } else { - res, err := giteaClient.ServeRawContent(o.generateUriClientArgs()) - log.Debug().Msg("acquisition") - - // Handle errors - if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) { - if o.TryIndexPages { - // copy the o struct & try if an index page exists - optionsForIndexPages := *o - optionsForIndexPages.TryIndexPages = false - optionsForIndexPages.appendTrailingSlash = true - for _, indexPage := range upstreamIndexPages { - optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaClient) { - return true - } - } - // compatibility fix for GitHub Pages (/example → /example.html) - optionsForIndexPages.appendTrailingSlash = false - optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html" - optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaClient) { - return true - } - } - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) - if o.TryIndexPages { - // copy the o struct & try if a not found page exists - optionsForNotFoundPages := *o - optionsForNotFoundPages.TryIndexPages = false - optionsForNotFoundPages.appendTrailingSlash = false - for _, notFoundPage := range upstreamNotFoundPages { - optionsForNotFoundPages.TargetPath = "/" + notFoundPage - if optionsForNotFoundPages.Upstream(ctx, giteaClient) { - return true - } - } - } - return false - } - if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) { - fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode()) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - - // Append trailing slash if missing (for index files), and redirect to fix filenames in general - // o.appendTrailingSlash is only true when looking for index pages - if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) { - ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect) - return true - } - if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) { - ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect) - return true - } - if o.redirectIfExists != "" { - ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect) - return true - } - log.Debug().Msg("error handling") - - // Set the MIME type - mimeType := o.getMimeTypeByExtension() - ctx.Response.Header.SetContentType(mimeType) - - // Set ETag - if cachedResponse.Exists { - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) - } else if res != nil { - cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag) - ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag) - } - - if ctx.Response.StatusCode() != fasthttp.StatusNotFound { - // Everything's okay so far - ctx.Response.SetStatusCode(fasthttp.StatusOK) - } - ctx.Response.Header.SetLastModified(o.BranchTimestamp) - - log.Debug().Msg("response preparations") - - // Write the response body to the original request - - // fasthttp else will set "Content-Length: 0" - ctx.Response.SetBodyStream(&strings.Reader{}, -1) - - err = res.BodyWriteTo(ctx.Response.BodyWriter()) - if err != nil { - fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err) - html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError) - return true - } - log.Debug().Msg("response") - - return true -} diff --git a/server/upstream/upstream_std.go b/server/upstream/upstream_std.go deleted file mode 100644 index ec62f7e..0000000 --- a/server/upstream/upstream_std.go +++ /dev/null @@ -1,148 +0,0 @@ -//go:build !fasthttp - -package upstream - -import ( - "errors" - "io" - "net/http" - "strings" - "time" - - "github.com/rs/zerolog/log" - - "codeberg.org/codeberg/pages/html" - "codeberg.org/codeberg/pages/server/context" - "codeberg.org/codeberg/pages/server/gitea" -) - -const ( - headerContentType = "Content-Type" - headerETag = "ETag" - headerLastModified = "Last-Modified" - headerIfModifiedSince = "If-Modified-Since" -) - -// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) { - log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() - - // Check if the branch exists and when it was modified - if o.BranchTimestamp.IsZero() { - branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch) - - if branch == nil { - html.ReturnErrorPage(ctx, "", http.StatusFailedDependency) - return true - } - o.TargetBranch = branch.Branch - o.BranchTimestamp = branch.Timestamp - } - - if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" { - html.ReturnErrorPage(ctx, "", http.StatusBadRequest) - return true - } - - // Check if the browser has a cached version - if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil { - if !ifModifiedSince.Before(o.BranchTimestamp) { - ctx.RespWriter.WriteHeader(http.StatusNotModified) - return true - } - } - log.Debug().Msg("preparations") - - reader, res, err := giteaClient.ServeRawContent(o.generateUriClientArgs()) - log.Debug().Msg("acquisition") - - // Handle errors - if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil) { - if o.TryIndexPages { - // copy the o struct & try if an index page exists - optionsForIndexPages := *o - optionsForIndexPages.TryIndexPages = false - optionsForIndexPages.appendTrailingSlash = true - for _, indexPage := range upstreamIndexPages { - optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage - if optionsForIndexPages.Upstream(ctx, giteaClient) { - return true - } - } - // compatibility fix for GitHub Pages (/example → /example.html) - optionsForIndexPages.appendTrailingSlash = false - optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html" - optionsForIndexPages.TargetPath = o.TargetPath + ".html" - if optionsForIndexPages.Upstream(ctx, giteaClient) { - return true - } - } - - ctx.Response().StatusCode = http.StatusNotFound - if o.TryIndexPages { - // copy the o struct & try if a not found page exists - optionsForNotFoundPages := *o - optionsForNotFoundPages.TryIndexPages = false - optionsForNotFoundPages.appendTrailingSlash = false - for _, notFoundPage := range upstreamNotFoundPages { - optionsForNotFoundPages.TargetPath = "/" + notFoundPage - if optionsForNotFoundPages.Upstream(ctx, giteaClient) { - return true - } - } - } - return false - } - if res != nil && (err != nil || res.StatusCode != http.StatusOK) { - log.Printf("Couldn't fetch contents (status code %d): %v\n", res.StatusCode, err) - html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) - return true - } - - // Append trailing slash if missing (for index files), and redirect to fix filenames in general - // o.appendTrailingSlash is only true when looking for index pages - if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") { - ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect) - return true - } - if strings.HasSuffix(ctx.Path(), "/index.html") { - ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect) - return true - } - if o.redirectIfExists != "" { - ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect) - return true - } - log.Debug().Msg("error handling") - - // Set the MIME type - mimeType := o.getMimeTypeByExtension() - ctx.Response().Header.Set(headerContentType, mimeType) - - // Set ETag - if res != nil { - ctx.Response().Header.Set(headerETag, res.Header.Get(headerETag)) - } - - if ctx.Response().StatusCode != http.StatusNotFound { - // Everything's okay so far - ctx.Response().StatusCode = http.StatusOK - } - ctx.Response().Header.Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123)) - - log.Debug().Msg("response preparations") - - // Write the response body to the original request - if reader != nil { - _, err := io.Copy(ctx.RespWriter, reader) - if err != nil { - log.Printf("Couldn't write body: %s\n", err) - html.ReturnErrorPage(ctx, "", http.StatusInternalServerError) - return true - } - } - - log.Debug().Msg("response") - - return true -}