From b9966487f6f7a9fc88ae285c7350917e455f27a0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 12 Nov 2022 20:37:20 +0100 Subject: [PATCH 1/2] switch to std http implementation instead of fasthttp (#106) close #100 close #109 close #113 close #28 close #63 Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/106 --- cmd/main.go | 41 ++-- go.mod | 17 +- go.sum | 33 +-- html/404.html | 8 +- html/error.go | 47 +++-- html/error.html | 38 ++++ html/html.go | 5 +- integration/get_test.go | 33 ++- server/certificates/certificates.go | 36 ++-- server/context/context.go | 62 ++++++ server/database/mock.go | 2 +- server/dns/const.go | 6 - server/dns/dns.go | 4 + server/gitea/cache.go | 112 +++++++++- server/gitea/client.go | 308 ++++++++++++++++++++-------- server/gitea/client_test.go | 23 --- server/gitea/fasthttp.go | 15 -- server/handler.go | 186 +++++++++-------- server/handler_test.go | 36 ++-- server/helpers.go | 8 +- server/setup.go | 54 ++--- server/try.go | 21 +- server/upstream/const.go | 24 --- server/upstream/domains.go | 10 + server/upstream/helper.go | 74 ++----- server/upstream/upstream.go | 194 +++++++++--------- server/utils/utils.go | 8 +- server/utils/utils_test.go | 6 +- 28 files changed, 827 insertions(+), 584 deletions(-) create mode 100644 html/error.html create mode 100644 server/context/context.go delete mode 100644 server/dns/const.go delete mode 100644 server/gitea/client_test.go delete mode 100644 server/gitea/fasthttp.go delete mode 100644 server/upstream/const.go diff --git a/cmd/main.go b/cmd/main.go index 41809cb..a3a61e1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,12 +1,12 @@ package cmd import ( - "bytes" "context" "crypto/tls" "errors" "fmt" "net" + "net/http" "os" "strings" "time" @@ -24,15 +24,15 @@ import ( // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. // TODO: make it a flag -var AllowedCorsDomains = [][]byte{ - []byte("fonts.codeberg.org"), - []byte("design.codeberg.org"), +var AllowedCorsDomains = []string{ + "fonts.codeberg.org", + "design.codeberg.org", } // BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. // TODO: Make it a flag too -var BlacklistedPaths = [][]byte{ - []byte("/.well-known/acme-challenge/"), +var BlacklistedPaths = []string{ + "/.well-known/acme-challenge/", } // Serve sets up and starts the web server. @@ -47,7 +47,7 @@ func Serve(ctx *cli.Context) error { giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/") giteaAPIToken := ctx.String("gitea-api-token") rawDomain := ctx.String("raw-domain") - mainDomainSuffix := []byte(ctx.String("pages-domain")) + mainDomainSuffix := ctx.String("pages-domain") rawInfoPage := ctx.String("raw-info-page") listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port")) enableHTTPServer := ctx.Bool("enable-http-server") @@ -65,12 +65,12 @@ func Serve(ctx *cli.Context) error { allowedCorsDomains := AllowedCorsDomains if len(rawDomain) != 0 { - allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain)) + allowedCorsDomains = append(allowedCorsDomains, rawDomain) } // Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash - if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) { - mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...) + if !strings.HasPrefix(mainDomainSuffix, ".") { + mainDomainSuffix = "." + mainDomainSuffix } keyCache := cache.NewKeyValueCache() @@ -79,26 +79,22 @@ func Serve(ctx *cli.Context) error { canonicalDomainCache := cache.NewKeyValueCache() // dnsLookupCache stores DNS lookups for custom domains dnsLookupCache := cache.NewKeyValueCache() - // branchTimestampCache stores branch timestamps for faster cache checking - branchTimestampCache := cache.NewKeyValueCache() - // fileResponseCache stores responses from the Gitea server - // TODO: make this an MRU cache with a size limit - fileResponseCache := cache.NewKeyValueCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewKeyValueCache() - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) + giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) if err != nil { return fmt.Errorf("could not create new gitea client: %v", err) } // Create handler based on settings - handler := server.Handler(mainDomainSuffix, []byte(rawDomain), + httpsHandler := server.Handler(mainDomainSuffix, rawDomain, giteaClient, giteaRoot, rawInfoPage, BlacklistedPaths, allowedCorsDomains, - dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache) + dnsLookupCache, canonicalDomainCache) - fastServer := server.SetupServer(handler) - httpServer := server.SetupHTTPACMEChallengeServer(challengeCache) + httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache) // Setup listener and TLS log.Info().Msgf("Listening on https://%s", listeningAddress) @@ -138,7 +134,7 @@ func Serve(ctx *cli.Context) error { if enableHTTPServer { go func() { log.Info().Msg("Start HTTP server listening on :80") - err := httpServer.ListenAndServe("[::]:80") + err := http.ListenAndServe("[::]:80", httpHandler) if err != nil { log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") } @@ -147,8 +143,7 @@ func Serve(ctx *cli.Context) error { // Start the web fastServer log.Info().Msgf("Start listening on %s", listener.Addr()) - err = fastServer.Serve(listener) - if err != nil { + if err := http.Serve(listener, httpsHandler); err != nil { log.Panic().Err(err).Msg("Couldn't start fastServer") } diff --git a/go.mod b/go.mod index 479c328..77ed762 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module codeberg.org/codeberg/pages -go 1.18 +go 1.19 require ( + code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a github.com/akrylysov/pogreb v0.10.1 github.com/go-acme/lego/v4 v4.5.3 @@ -11,8 +12,6 @@ require ( github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 - github.com/valyala/fasthttp v1.31.0 - github.com/valyala/fastjson v1.6.3 ) require ( @@ -31,7 +30,6 @@ require ( github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect - github.com/andybalholm/brotli v1.0.2 // indirect github.com/aws/aws-sdk-go v1.39.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect @@ -39,6 +37,7 @@ require ( github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dnsimple/dnsimple-go v0.70.1 // indirect @@ -46,6 +45,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/go-errors/errors v1.0.1 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect @@ -57,13 +57,13 @@ require ( github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect github.com/jarcoal/httpmock v1.0.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.7 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect - github.com/klauspost/compress v1.13.4 // indirect github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect @@ -104,15 +104,14 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect github.com/vultr/govultr/v2 v2.7.1 // indirect go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/api v0.20.0 // indirect diff --git a/go.sum b/go.sum index 23a58bc..a44001c 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk= +code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -66,8 +68,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbxuNdz3FeyvHFnRjmo= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= -github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -106,6 +106,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/deepmap/oapi-codegen v1.6.1 h1:2BvsmRb6pogGNtr8Ann+esAbSKFXx2CZN18VpAMecnw= github.com/deepmap/oapi-codegen v1.6.1/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -136,6 +138,8 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -182,7 +186,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -243,6 +246,9 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -282,8 +288,6 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcM github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc= github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -494,15 +498,9 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE= -github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= -github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 h1:TFXGGMHmml4rs29PdPisC/aaCzOxUu1Vsh9on/IpUfE= github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14/go.mod h1:RWc47jtnVuQv6+lY3c768WtXCas/Xi+U5UFc5xULmYg= github.com/vultr/govultr/v2 v2.7.1 h1:uF9ERet++Gb+7Cqs3p1P6b6yebeaZqVd7t5P2uZCaJU= @@ -539,8 +537,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -604,8 +604,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -667,12 +667,13 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/html/404.html b/html/404.html index 21d968e..7c721b5 100644 --- a/html/404.html +++ b/html/404.html @@ -3,7 +3,7 @@
-.domains
file"
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
+// with "%status%" and %message% replaced with the provided statusCode and msg
+func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
+ ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
+ ctx.RespWriter.WriteHeader(statusCode)
+
+ if msg == "" {
+ msg = errorBody(statusCode)
+ } else {
+ // TODO: use template engine
+ msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode))
}
- if code == fasthttp.StatusFailedDependency {
+
+ _, _ = ctx.RespWriter.Write([]byte(msg))
+}
+
+func errorMessage(statusCode int) string {
+ message := http.StatusText(statusCode)
+
+ switch statusCode {
+ case http.StatusMisdirectedRequest:
+ message += " - domain not specified in .domains
file"
+ case http.StatusFailedDependency:
message += " - target repo/branch doesn't exist or is private"
}
- // TODO: use template engine?
- ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
+
+ return message
+}
+
+// TODO: use template engine
+func errorBody(statusCode int) string {
+ return strings.ReplaceAll(NotFoundPage,
+ "%status%",
+ strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
}
diff --git a/html/error.html b/html/error.html
new file mode 100644
index 0000000..f1975f7
--- /dev/null
+++ b/html/error.html
@@ -0,0 +1,38 @@
+
+
+
+
+
+ .domains
file", http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
if targetOwner != "" {
- ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect)
return
}
- log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
return
}
- log.Info().Msg("tryBranch, now trying upstream 7")
+ log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
}
- log.Warn().Msg("Couldn't handle request, none of the options succeed")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
return
}
}
diff --git a/server/handler_test.go b/server/handler_test.go
index f9a721a..c0aca14 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -1,44 +1,42 @@
package server
import (
- "fmt"
+ "net/http/httptest"
"testing"
"time"
- "github.com/valyala/fasthttp"
-
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
+ "github.com/rs/zerolog/log"
)
func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
- giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
+ giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false)
testHandler := Handler(
- []byte("codeberg.page"), []byte("raw.codeberg.org"),
+ "codeberg.page", "raw.codeberg.org",
giteaClient,
giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
- [][]byte{[]byte("/.well-known/acme-challenge/")},
- [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
- cache.NewKeyValueCache(),
- cache.NewKeyValueCache(),
+ []string{"/.well-known/acme-challenge/"},
+ []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)
testCase := func(uri string, status int) {
- ctx := &fasthttp.RequestCtx{
- Request: *fasthttp.AcquireRequest(),
- Response: *fasthttp.AcquireResponse(),
- }
- ctx.Request.SetRequestURI(uri)
- fmt.Printf("Start: %v\n", time.Now())
+ req := httptest.NewRequest("GET", uri, nil)
+ w := httptest.NewRecorder()
+
+ log.Printf("Start: %v\n", time.Now())
start := time.Now()
- testHandler(ctx)
+ testHandler(w, req)
end := time.Now()
- fmt.Printf("Done: %v\n", time.Now())
- if ctx.Response.StatusCode() != status {
- t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
+ log.Printf("Done: %v\n", time.Now())
+
+ resp := w.Result()
+
+ if resp.StatusCode != status {
+ t.Errorf("request failed with status code %d", resp.StatusCode)
} else {
t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
}
diff --git a/server/helpers.go b/server/helpers.go
index 6d55ddf..7c898cd 100644
--- a/server/helpers.go
+++ b/server/helpers.go
@@ -1,13 +1,13 @@
package server
import (
- "bytes"
+ "strings"
)
-// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
+// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
// string for custom domains.
-func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
- if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
+func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string {
+ if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) {
return "max-age=63072000; includeSubdomains; preload"
} else {
return ""
diff --git a/server/setup.go b/server/setup.go
index 176bb42..e7194ed 100644
--- a/server/setup.go
+++ b/server/setup.go
@@ -1,53 +1,27 @@
package server
import (
- "bytes"
- "fmt"
"net/http"
- "time"
-
- "github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
+ "strings"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/utils"
)
-type fasthttpLogger struct{}
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
+ challengePath := "/.well-known/acme-challenge/"
-func (fasthttpLogger) Printf(format string, args ...interface{}) {
- log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
-}
-
-func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
- // Enable compression by wrapping the handler with the compression function provided by FastHTTP
- compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
-
- return &fasthttp.Server{
- Handler: compressedHandler,
- DisablePreParseMultipartForm: true,
- NoDefaultServerHeader: true,
- NoDefaultDate: true,
- ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
- Logger: fasthttpLogger{},
- }
-}
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
- challengePath := []byte("/.well-known/acme-challenge/")
-
- return &fasthttp.Server{
- Handler: func(ctx *fasthttp.RequestCtx) {
- if bytes.HasPrefix(ctx.Path(), challengePath) {
- challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
- if !ok || challenge == nil {
- ctx.SetStatusCode(http.StatusNotFound)
- ctx.SetBodyString("no challenge for this token")
- }
- ctx.SetBodyString(challenge.(string))
- } else {
- ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
+ return func(w http.ResponseWriter, req *http.Request) {
+ ctx := context.New(w, req)
+ if strings.HasPrefix(ctx.Path(), challengePath) {
+ challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + string(strings.TrimPrefix(ctx.Path(), challengePath)))
+ if !ok || challenge == nil {
+ ctx.String("no challenge for this token", http.StatusNotFound)
}
- },
+ ctx.String(challenge.(string))
+ } else {
+ ctx.Redirect("https://"+string(ctx.Host())+string(ctx.Path()), http.StatusMovedPermanently)
+ }
}
}
diff --git a/server/try.go b/server/try.go
index 24831c4..135c1e0 100644
--- a/server/try.go
+++ b/server/try.go
@@ -1,38 +1,37 @@
package server
import (
- "bytes"
+ "net/http"
"strings"
- "github.com/valyala/fasthttp"
-
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
- mainDomainSuffix, trimmedHost []byte,
+func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix, trimmedHost string,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath string,
- canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+ canonicalDomainCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
- if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+ if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
- canonicalPath := string(ctx.RequestURI())
+ canonicalPath := ctx.Req.RequestURI
if targetRepo != "pages" {
path := strings.SplitN(canonicalPath, "/", 3)
if len(path) >= 3 {
canonicalPath = "/" + path[2]
}
}
- ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
return
}
}
@@ -44,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions.Host = string(trimmedHost)
// Try to request the file from the Gitea API
- if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
+ if !targetOptions.Upstream(ctx, giteaClient) {
+ html.ReturnErrorPage(ctx, "", ctx.StatusCode)
}
}
diff --git a/server/upstream/const.go b/server/upstream/const.go
deleted file mode 100644
index 247e1d1..0000000
--- a/server/upstream/const.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package upstream
-
-import "time"
-
-// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
-var defaultBranchCacheTimeout = 15 * time.Minute
-
-// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
-// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
-// picked up faster, while still allowing the content to be cached longer if nothing changes.
-var branchExistenceCacheTimeout = 5 * time.Minute
-
-// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
-// on your available memory.
-// TODO: move as option into cache interface
-var fileCacheTimeout = 5 * time.Minute
-
-// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-var fileCacheSizeLimit = 1024 * 1024
-
-// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
-var canonicalDomainCacheTimeout = 15 * time.Minute
-
-const canonicalDomainConfig = ".domains"
diff --git a/server/upstream/domains.go b/server/upstream/domains.go
index 553c148..6ad6506 100644
--- a/server/upstream/domains.go
+++ b/server/upstream/domains.go
@@ -2,11 +2,19 @@ package upstream
import (
"strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
+// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
+var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"
+
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
@@ -36,6 +44,8 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
valid = true
}
}
+ } else {
+ log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo)
}
domains = append(domains, targetOwner+mainDomainSuffix)
if domains[len(domains)-1] == actualDomain {
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
index 28f4474..6bc23c8 100644
--- a/server/upstream/helper.go
+++ b/server/upstream/helper.go
@@ -1,84 +1,36 @@
package upstream
import (
- "mime"
- "path"
- "strconv"
- "strings"
- "time"
+ "errors"
- "codeberg.org/codeberg/pages/server/cache"
- "codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
-)
-type branchTimestamp struct {
- Branch string
- Timestamp time.Time
-}
+ "codeberg.org/codeberg/pages/server/gitea"
+)
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
-func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp {
log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
- if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
- if result == nil {
- log.Debug().Msg("branchTimestampCache found item, but result is empty")
- return nil
- }
- log.Debug().Msg("branchTimestampCache found item, returning result")
- return result.(*branchTimestamp)
- }
- result := &branchTimestamp{
- Branch: branch,
- }
+
if len(branch) == 0 {
// Get default branch
defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
if err != nil {
log.Err(err).Msg("Could't fetch default branch from repository")
- _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
return nil
}
- log.Debug().Msg("Succesfully fetched default branch from Gitea")
- result.Branch = defaultBranch
+ log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
+ branch = defaultBranch
}
- timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+ timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch)
if err != nil {
- log.Err(err).Msg("Could not get latest commit's timestamp from branch")
+ if !errors.Is(err, gitea.ErrorNotFound) {
+ log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch")
+ }
return nil
}
- log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache")
- result.Timestamp = timestamp
- _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
- return result
-}
-
-func (o *Options) getMimeTypeByExtension() string {
- if o.ForbiddenMimeTypes == nil {
- o.ForbiddenMimeTypes = make(map[string]bool)
- }
- mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
- mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
- if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
- if o.DefaultMimeType != "" {
- mimeType = o.DefaultMimeType
- } else {
- mimeType = "application/octet-stream"
- }
- }
- return mimeType
-}
-
-func (o *Options) generateUri() string {
- return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
-}
-
-func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
- return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
-}
-
-func (o *Options) timestamp() string {
- return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
+ log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp)
+ return timestamp
}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index 61c90de..d37c35e 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -1,20 +1,27 @@
package upstream
import (
- "bytes"
"errors"
+ "fmt"
"io"
+ "net/http"
"strings"
"time"
"github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
- "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
)
+const (
+ headerLastModified = "Last-Modified"
+ headerIfModifiedSince = "If-Modified-Since"
+
+ rawMime = "text/plain; charset=utf-8"
+)
+
// upstreamIndexPages lists pages that may be considered as index pages for directories.
var upstreamIndexPages = []string{
"index.html",
@@ -35,61 +42,61 @@ type Options struct {
// Used for debugging purposes.
Host string
- DefaultMimeType string
- ForbiddenMimeTypes map[string]bool
- TryIndexPages bool
- BranchTimestamp time.Time
+ TryIndexPages bool
+ BranchTimestamp time.Time
// internal
appendTrailingSlash bool
redirectIfExists string
+
+ ServeRaw bool
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
- log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
+ log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+
+ if o.TargetOwner == "" || o.TargetRepo == "" {
+ html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
+ return true
+ }
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
- branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
+ branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
- if branch == nil {
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ if branch == nil || branch.Branch == "" {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
+ http.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
- if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
- html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
- return true
- }
-
// Check if the browser has a cached version
- if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
- if !ifModifiedSince.Before(o.BranchTimestamp) {
- ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
- return true
+ if ctx.Response() != nil {
+ if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil {
+ if !ifModifiedSince.Before(o.BranchTimestamp) {
+ ctx.RespWriter.WriteHeader(http.StatusNotModified)
+ log.Trace().Msg("check response against last modified: valid")
+ return true
+ }
}
+ log.Trace().Msg("check response against last modified: outdated")
}
log.Debug().Msg("Preparing")
- // Make a GET request to the upstream URL
- uri := o.generateUri()
- var res *fasthttp.Response
- var cachedResponse gitea.FileResponse
- var err error
- if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
- cachedResponse = cachedValue.(gitea.FileResponse)
- } else {
- res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
+ reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath)
+ if reader != nil {
+ defer reader.Close()
}
log.Debug().Msg("Aquisting")
- // Handle errors
- if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
+ // Handle not found error
+ if err != nil && errors.Is(err, gitea.ErrorNotFound) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
@@ -97,25 +104,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
- if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
- optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
+ optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
- if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
- ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+
+ ctx.StatusCode = http.StatusNotFound
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
@@ -123,94 +125,84 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
- if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
return true
}
}
}
- if res != nil {
- // Update cache if the request is fresh
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
- }
return false
}
- if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
- log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
- html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+
+ // handle unexpected client errors
+ if err != nil || reader == nil || statusCode != http.StatusOK {
+ log.Debug().Msg("Handling error")
+ var msg string
+
+ if err != nil {
+ msg = "gitea client returned unexpected error"
+ log.Error().Err(err).Msg(msg)
+ msg = fmt.Sprintf("%s: %v", msg, err)
+ }
+ if reader == nil {
+ msg = "gitea client returned no reader"
+ log.Error().Msg(msg)
+ }
+ if statusCode != http.StatusOK {
+ msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+ log.Error().Msg(msg)
+ }
+
+ html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
- if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
- ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
+ if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
+ ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
return true
}
- if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
- ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
+ if strings.HasSuffix(ctx.Path(), "/index.html") {
+ ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
- ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect)
return true
}
- log.Debug().Msg("Handling error")
-
- // Set the MIME type
- mimeType := o.getMimeTypeByExtension()
- ctx.Response.Header.SetContentType(mimeType)
-
- // Set ETag
- if cachedResponse.Exists {
- ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
- } else if res != nil {
- cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
- ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+ // Set ETag & MIME
+ if eTag := header.Get(gitea.ETagHeader); eTag != "" {
+ ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
}
-
- if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
- // Everything's okay so far
- ctx.Response.SetStatusCode(fasthttp.StatusOK)
+ if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
+ ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
}
- ctx.Response.Header.SetLastModified(o.BranchTimestamp)
+ if length := header.Get(gitea.ContentLengthHeader); length != "" {
+ ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
+ }
+ if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
+ } else {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
+ }
+ ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
log.Debug().Msg("Prepare response")
- // Write the response body to the original request
- var cacheBodyWriter bytes.Buffer
- if res != nil {
- if res.Header.ContentLength() > fileCacheSizeLimit {
- // fasthttp else will set "Content-Length: 0"
- ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+ ctx.RespWriter.WriteHeader(ctx.StatusCode)
- err = res.BodyWriteTo(ctx.Response.BodyWriter())
- } else {
- // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
- err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
+ // Write the response body to the original request
+ if reader != nil {
+ _, err := io.Copy(ctx.RespWriter, reader)
+ if err != nil {
+ log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
+ html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
+ return true
}
- } else {
- _, err = ctx.Write(cachedResponse.Body)
- }
- if err != nil {
- log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
- html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
- return true
}
log.Debug().Msg("Sending response")
- if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
- cachedResponse.Exists = true
- cachedResponse.MimeType = mimeType
- cachedResponse.Body = cacheBodyWriter.Bytes()
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
- }
-
return true
}
diff --git a/server/utils/utils.go b/server/utils/utils.go
index 7be330f..30f948d 100644
--- a/server/utils/utils.go
+++ b/server/utils/utils.go
@@ -1,9 +1,11 @@
package utils
-import "bytes"
+import (
+ "strings"
+)
-func TrimHostPort(host []byte) []byte {
- i := bytes.IndexByte(host, ':')
+func TrimHostPort(host string) string {
+ i := strings.IndexByte(host, ':')
if i >= 0 {
return host[:i]
}
diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go
index 3dc0632..2532392 100644
--- a/server/utils/utils_test.go
+++ b/server/utils/utils_test.go
@@ -7,7 +7,7 @@ import (
)
func TestTrimHostPort(t *testing.T) {
- assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
- assert.EqualValues(t, "", TrimHostPort([]byte(":")))
- assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
+ assert.EqualValues(t, "aa", TrimHostPort("aa"))
+ assert.EqualValues(t, "", TrimHostPort(":"))
+ assert.EqualValues(t, "example.com", TrimHostPort("example.com:80"))
}
From 6c63b66ce44535cb2fdc3543780e01fa9994e3d5 Mon Sep 17 00:00:00 2001
From: 6543 <6543@obermui.de>
Date: Sat, 12 Nov 2022 20:43:44 +0100
Subject: [PATCH 2/2] Refactor split long functions (#135)
we have big functions that handle all stuff ... we should split this into smaler chuncks so we could test them seperate and make clear cuts in what happens where
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/135
---
cmd/main.go | 5 +-
server/certificates/certificates.go | 8 +-
server/context/context.go | 10 +-
server/gitea/client.go | 8 +
server/handler.go | 321 ------------------------
server/handler/handler.go | 111 ++++++++
server/handler/handler_custom_domain.go | 71 ++++++
server/handler/handler_raw_domain.go | 67 +++++
server/handler/handler_sub_domain.go | 120 +++++++++
server/{ => handler}/handler_test.go | 7 +-
server/{helpers.go => handler/hsts.go} | 2 +-
server/handler/try.go | 76 ++++++
server/try.go | 49 ----
server/upstream/domains.go | 16 +-
server/upstream/header.go | 28 +++
server/upstream/helper.go | 33 ++-
server/upstream/upstream.go | 42 ++--
17 files changed, 547 insertions(+), 427 deletions(-)
delete mode 100644 server/handler.go
create mode 100644 server/handler/handler.go
create mode 100644 server/handler/handler_custom_domain.go
create mode 100644 server/handler/handler_raw_domain.go
create mode 100644 server/handler/handler_sub_domain.go
rename server/{ => handler}/handler_test.go (85%)
rename server/{helpers.go => handler/hsts.go} (96%)
create mode 100644 server/handler/try.go
delete mode 100644 server/try.go
create mode 100644 server/upstream/header.go
diff --git a/cmd/main.go b/cmd/main.go
index a3a61e1..6ad1aa8 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -20,6 +20,7 @@ import (
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/handler"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -88,9 +89,9 @@ func Serve(ctx *cli.Context) error {
}
// Create handler based on settings
- httpsHandler := server.Handler(mainDomainSuffix, rawDomain,
+ httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
- giteaRoot, rawInfoPage,
+ rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache)
diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go
index 429ab23..42620b1 100644
--- a/server/certificates/certificates.go
+++ b/server/certificates/certificates.go
@@ -78,8 +78,12 @@ func TLSConfig(mainDomainSuffix string,
// DNS not set up, return main certificate to redirect to the docs
sni = mainDomainSuffix
} else {
- _, _ = targetRepo, targetBranch
- _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, mainDomainSuffix, canonicalDomainCache)
+ targetOpt := &upstream.Options{
+ TargetOwner: targetOwner,
+ TargetRepo: targetRepo,
+ TargetBranch: targetBranch,
+ }
+ _, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
if !valid {
sni = mainDomainSuffix
}
diff --git a/server/context/context.go b/server/context/context.go
index be01df0..481fee2 100644
--- a/server/context/context.go
+++ b/server/context/context.go
@@ -3,6 +3,8 @@ package context
import (
stdContext "context"
"net/http"
+
+ "codeberg.org/codeberg/pages/server/utils"
)
type Context struct {
@@ -42,10 +44,6 @@ func (c *Context) String(raw string, status ...int) {
_, _ = c.RespWriter.Write([]byte(raw))
}
-func (c *Context) IsMethod(m string) bool {
- return c.Req.Method == m
-}
-
func (c *Context) Redirect(uri string, statusCode int) {
http.Redirect(c.RespWriter, c.Req, uri, statusCode)
}
@@ -60,3 +58,7 @@ func (c *Context) Path() string {
func (c *Context) Host() string {
return c.Req.URL.Host
}
+
+func (c *Context) TrimHostPort() string {
+ return utils.TrimHostPort(c.Req.Host)
+}
diff --git a/server/gitea/client.go b/server/gitea/client.go
index c63ee21..51647ba 100644
--- a/server/gitea/client.go
+++ b/server/gitea/client.go
@@ -45,6 +45,8 @@ type Client struct {
sdkClient *gitea.Client
responseCache cache.SetGetKey
+ giteaRoot string
+
followSymlinks bool
supportLFS bool
@@ -79,6 +81,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
sdkClient: sdk,
responseCache: respCache,
+ giteaRoot: giteaRoot,
+
followSymlinks: followSymlinks,
supportLFS: supportLFS,
@@ -87,6 +91,10 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
}, err
}
+func (client *Client) ContentWebLink(targetOwner, targetRepo, branch, resource string) string {
+ return path.Join(client.giteaRoot, targetOwner, targetRepo, "src/branch", branch, resource)
+}
+
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
if err != nil {
diff --git a/server/handler.go b/server/handler.go
deleted file mode 100644
index 894cd25..0000000
--- a/server/handler.go
+++ /dev/null
@@ -1,321 +0,0 @@
-package server
-
-import (
- "fmt"
- "net/http"
- "path"
- "strings"
-
- "github.com/rs/zerolog"
- "github.com/rs/zerolog/log"
-
- "codeberg.org/codeberg/pages/html"
- "codeberg.org/codeberg/pages/server/cache"
- "codeberg.org/codeberg/pages/server/context"
- "codeberg.org/codeberg/pages/server/dns"
- "codeberg.org/codeberg/pages/server/gitea"
- "codeberg.org/codeberg/pages/server/upstream"
- "codeberg.org/codeberg/pages/server/utils"
- "codeberg.org/codeberg/pages/server/version"
-)
-
-const (
- headerAccessControlAllowOrigin = "Access-Control-Allow-Origin"
- headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
-)
-
-// Handler handles a single HTTP request to the web server.
-func Handler(mainDomainSuffix, rawDomain string,
- giteaClient *gitea.Client,
- giteaRoot, rawInfoPage string,
- blacklistedPaths, allowedCorsDomains []string,
- dnsLookupCache, canonicalDomainCache cache.SetGetKey,
-) http.HandlerFunc {
- return func(w http.ResponseWriter, req *http.Request) {
- log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger()
- ctx := context.New(w, req)
-
- ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
-
- // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
- ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
-
- // Enable browser caching for up to 10 minutes
- ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600")
-
- trimmedHost := utils.TrimHostPort(req.Host)
-
- // Add HSTS for RawDomain and MainDomainSuffix
- if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
- ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
- }
-
- // Block all methods not required for static pages
- if !ctx.IsMethod(http.MethodGet) && !ctx.IsMethod(http.MethodHead) && !ctx.IsMethod(http.MethodOptions) {
- ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1
- ctx.String("Method not allowed", http.StatusMethodNotAllowed)
- return
- }
-
- // Block blacklisted paths (like ACME challenges)
- for _, blacklistedPath := range blacklistedPaths {
- if strings.HasPrefix(ctx.Path(), blacklistedPath) {
- html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
- return
- }
- }
-
- // Allow CORS for specified domains
- allowCors := false
- for _, allowedCorsDomain := range allowedCorsDomains {
- if strings.EqualFold(trimmedHost, allowedCorsDomain) {
- allowCors = true
- break
- }
- }
- if allowCors {
- ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*")
- ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead)
- }
-
- ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1
- if ctx.IsMethod(http.MethodOptions) {
- ctx.RespWriter.WriteHeader(http.StatusNoContent)
- return
- }
-
- // Prepare request information to Gitea
- var targetOwner, targetRepo, targetBranch, targetPath string
- targetOptions := &upstream.Options{
- TryIndexPages: true,
- }
-
- // tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
- // also disallow search indexing and add a Link header to the canonical URL.
- // TODO: move into external func to not alert vars indirectly
- tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool {
- if repo == "" {
- log.Debug().Msg("tryBranch: repo is empty")
- return false
- }
-
- // Replace "~" to "/" so we can access branch that contains slash character
- // Branch name cannot contain "~" so doing this is okay
- branch = strings.ReplaceAll(branch, "~", "/")
-
- // Check if the branch exists, otherwise treat it as a file path
- branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch)
- if branchTimestampResult == nil {
- log.Debug().Msg("tryBranch: branch doesn't exist")
- return false
- }
-
- // Branch exists, use it
- targetRepo = repo
- targetPath = path.Join(_path...)
- targetBranch = branchTimestampResult.Branch
-
- targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
-
- if canonicalLink != "" {
- // Hide from search machines & add canonical link
- ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex")
- ctx.RespWriter.Header().Set("Link",
- strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
- "; rel=\"canonical\"",
- )
- }
-
- log.Debug().Msg("tryBranch: true")
- return true
- }
-
- log.Debug().Msg("preparations")
- if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
- // Serve raw content from RawDomain
- log.Debug().Msg("raw domain")
-
- targetOptions.TryIndexPages = false
- targetOptions.ServeRaw = true
-
- pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
- if len(pathElements) < 2 {
- // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
- ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
- return
- }
- targetOwner = pathElements[0]
- targetRepo = pathElements[1]
-
- // raw.codeberg.org/example/myrepo/@main/index.html
- if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
- log.Debug().Msg("raw domain preparations, now trying with specified branch")
- if tryBranch(log,
- targetRepo, pathElements[2][1:], pathElements[3:],
- giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
- ) {
- log.Debug().Msg("tryBranch, now trying upstream 1")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- return
- }
- log.Debug().Msg("missing branch info")
- html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency)
- return
- }
-
- log.Debug().Msg("raw domain preparations, now trying with default branch")
- tryBranch(log,
- targetRepo, "", pathElements[2:],
- giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
- )
- log.Debug().Msg("tryBranch, now trying upstream 2")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- return
-
- } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
- // Serve pages from subdomains of MainDomainSuffix
- log.Debug().Msg("main domain suffix")
-
- pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
- targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix)
- targetRepo = pathElements[0]
- targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
-
- if targetOwner == "www" {
- // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
- ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect)
- return
- }
-
- // Check if the first directory is a repo with the second directory as a branch
- // example.codeberg.page/myrepo/@main/index.html
- if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
- if targetRepo == "pages" {
- // example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
- ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
- return
- }
-
- log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
- branch := pathElements[1][1:]
- if tryBranch(log,
- pathElements[0], branch, pathElements[2:],
- "/"+pathElements[0]+"/%p",
- ) {
- log.Debug().Msg("tryBranch, now trying upstream 3")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- } else {
- html.ReturnErrorPage(ctx,
- fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo),
- http.StatusFailedDependency)
- }
- return
- }
-
- // Check if the first directory is a branch for the "pages" repo
- // example.codeberg.page/@main/index.html
- if strings.HasPrefix(pathElements[0], "@") {
- log.Debug().Msg("main domain preparations, now trying with specified branch")
- branch := pathElements[0][1:]
- if tryBranch(log,
- "pages", branch, pathElements[1:], "/%p") {
- log.Debug().Msg("tryBranch, now trying upstream 4")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, "pages", targetBranch, targetPath,
- canonicalDomainCache)
- } else {
- html.ReturnErrorPage(ctx,
- fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"),
- http.StatusFailedDependency)
- }
- return
- }
-
- // Check if the first directory is a repo with a "pages" branch
- // example.codeberg.page/myrepo/index.html
- // example.codeberg.page/pages/... is not allowed here.
- log.Debug().Msg("main domain preparations, now trying with specified repo")
- if pathElements[0] != "pages" && tryBranch(log,
- pathElements[0], "pages", pathElements[1:], "") {
- log.Debug().Msg("tryBranch, now trying upstream 5")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- return
- }
-
- // Try to use the "pages" repo on its default branch
- // example.codeberg.page/index.html
- log.Debug().Msg("main domain preparations, now trying with default repo/branch")
- if tryBranch(log,
- "pages", "", pathElements, "") {
- log.Debug().Msg("tryBranch, now trying upstream 6")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- return
- }
-
- // Couldn't find a valid repo/branch
- html.ReturnErrorPage(ctx,
- fmt.Sprintf("couldn't find a valid repo[%s]/branch[%s]", targetRepo, targetBranch),
- http.StatusFailedDependency)
- return
- } else {
- trimmedHostStr := string(trimmedHost)
-
- // Serve pages from custom domains
- targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
- if targetOwner == "" {
- html.ReturnErrorPage(ctx,
- "could not obtain repo owner from custom domain",
- http.StatusFailedDependency)
- return
- }
-
- pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
- canonicalLink := ""
- if strings.HasPrefix(pathElements[0], "@") {
- targetBranch = pathElements[0][1:]
- pathElements = pathElements[1:]
- canonicalLink = "/%p"
- }
-
- // Try to use the given repo on the given branch or the default branch
- log.Debug().Msg("custom domain preparations, now trying with details from DNS")
- if tryBranch(log,
- targetRepo, targetBranch, pathElements, canonicalLink) {
- canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
- if !valid {
- html.ReturnErrorPage(ctx, "domain not specified in .domains
file", http.StatusMisdirectedRequest)
- return
- } else if canonicalDomain != trimmedHostStr {
- // only redirect if the target is also a codeberg page!
- targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
- if targetOwner != "" {
- ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect)
- return
- }
-
- html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
- return
- }
-
- log.Debug().Msg("tryBranch, now trying upstream 7")
- tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache)
- return
- }
-
- html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
- return
- }
- }
-}
diff --git a/server/handler/handler.go b/server/handler/handler.go
new file mode 100644
index 0000000..b42751e
--- /dev/null
+++ b/server/handler/handler.go
@@ -0,0 +1,111 @@
+package handler
+
+import (
+ "net/http"
+ "strings"
+
+ "github.com/rs/zerolog/log"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/version"
+)
+
+const (
+ headerAccessControlAllowOrigin = "Access-Control-Allow-Origin"
+ headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
+)
+
+// Handler handles a single HTTP request to the web server.
+func Handler(mainDomainSuffix, rawDomain string,
+ giteaClient *gitea.Client,
+ rawInfoPage string,
+ blacklistedPaths, allowedCorsDomains []string,
+ dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+) http.HandlerFunc {
+ return func(w http.ResponseWriter, req *http.Request) {
+ log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger()
+ ctx := context.New(w, req)
+
+ ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
+
+ // Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
+ ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
+
+ // Enable browser caching for up to 10 minutes
+ ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600")
+
+ trimmedHost := ctx.TrimHostPort()
+
+ // Add HSTS for RawDomain and MainDomainSuffix
+ if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+ ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
+ }
+
+ // Handle all http methods
+ ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions)
+ switch ctx.Req.Method {
+ case http.MethodOptions:
+ // return Allow header
+ ctx.RespWriter.WriteHeader(http.StatusNoContent)
+ return
+ case http.MethodGet,
+ http.MethodHead:
+ // end switch case and handle allowed requests
+ break
+ default:
+ // Block all methods not required for static pages
+ ctx.String("Method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+
+ // Block blacklisted paths (like ACME challenges)
+ for _, blacklistedPath := range blacklistedPaths {
+ if strings.HasPrefix(ctx.Path(), blacklistedPath) {
+ html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
+ return
+ }
+ }
+
+ // Allow CORS for specified domains
+ allowCors := false
+ for _, allowedCorsDomain := range allowedCorsDomains {
+ if strings.EqualFold(trimmedHost, allowedCorsDomain) {
+ allowCors = true
+ break
+ }
+ }
+ if allowCors {
+ ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*")
+ ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead)
+ }
+
+ // Prepare request information to Gitea
+ pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
+
+ if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
+ log.Debug().Msg("raw domain request detecded")
+ handleRaw(log, ctx, giteaClient,
+ mainDomainSuffix, rawInfoPage,
+ trimmedHost,
+ pathElements,
+ canonicalDomainCache)
+ } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+ log.Debug().Msg("subdomain request detecded")
+ handleSubDomain(log, ctx, giteaClient,
+ mainDomainSuffix,
+ trimmedHost,
+ pathElements,
+ canonicalDomainCache)
+ } else {
+ log.Debug().Msg("custom domain request detecded")
+ handleCustomDomain(log, ctx, giteaClient,
+ mainDomainSuffix,
+ trimmedHost,
+ pathElements,
+ dnsLookupCache, canonicalDomainCache)
+ }
+ }
+}
diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go
new file mode 100644
index 0000000..bec3b46
--- /dev/null
+++ b/server/handler/handler_custom_domain.go
@@ -0,0 +1,71 @@
+package handler
+
+import (
+ "net/http"
+ "path"
+ "strings"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/dns"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/upstream"
+ "github.com/rs/zerolog"
+)
+
+func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix string,
+ trimmedHost string,
+ pathElements []string,
+ dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+) {
+ // Serve pages from custom domains
+ targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, string(mainDomainSuffix), dnsLookupCache)
+ if targetOwner == "" {
+ html.ReturnErrorPage(ctx,
+ "could not obtain repo owner from custom domain",
+ http.StatusFailedDependency)
+ return
+ }
+
+ pathParts := pathElements
+ canonicalLink := false
+ if strings.HasPrefix(pathElements[0], "@") {
+ targetBranch = pathElements[0][1:]
+ pathParts = pathElements[1:]
+ canonicalLink = true
+ }
+
+ // Try to use the given repo on the given branch or the default branch
+ log.Debug().Msg("custom domain preparations, now trying with details from DNS")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: true,
+ TargetOwner: targetOwner,
+ TargetRepo: targetRepo,
+ TargetBranch: targetBranch,
+ TargetPath: path.Join(pathParts...),
+ }, canonicalLink); works {
+ canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, string(mainDomainSuffix), canonicalDomainCache)
+ if !valid {
+ html.ReturnErrorPage(ctx, "domain not specified in .domains
file", http.StatusMisdirectedRequest)
+ return
+ } else if canonicalDomain != trimmedHost {
+ // only redirect if the target is also a codeberg page!
+ targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
+ if targetOwner != "" {
+ ctx.Redirect("https://"+canonicalDomain+string(targetOpt.TargetPath), http.StatusTemporaryRedirect)
+ return
+ }
+
+ html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
+ return
+ }
+
+ log.Debug().Msg("tryBranch, now trying upstream 7")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ return
+ }
+
+ html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
+}
diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go
new file mode 100644
index 0000000..5e974da
--- /dev/null
+++ b/server/handler/handler_raw_domain.go
@@ -0,0 +1,67 @@
+package handler
+
+import (
+ "fmt"
+ "net/http"
+ "path"
+ "strings"
+
+ "github.com/rs/zerolog"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/upstream"
+)
+
+func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix, rawInfoPage string,
+ trimmedHost string,
+ pathElements []string,
+ canonicalDomainCache cache.SetGetKey,
+) {
+ // Serve raw content from RawDomain
+ log.Debug().Msg("raw domain")
+
+ if len(pathElements) < 2 {
+ // https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
+ ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
+ return
+ }
+
+ // raw.codeberg.org/example/myrepo/@main/index.html
+ if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
+ log.Debug().Msg("raw domain preparations, now trying with specified branch")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ ServeRaw: true,
+ TargetOwner: pathElements[0],
+ TargetRepo: pathElements[1],
+ TargetBranch: pathElements[2][1:],
+ TargetPath: path.Join(pathElements[3:]...),
+ }, true); works {
+ log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ return
+ }
+ log.Debug().Msg("missing branch info")
+ html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency)
+ return
+ }
+
+ log.Debug().Msg("raw domain preparations, now trying with default branch")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: false,
+ ServeRaw: true,
+ TargetOwner: pathElements[0],
+ TargetRepo: pathElements[1],
+ TargetPath: path.Join(pathElements[2:]...),
+ }, true); works {
+ log.Trace().Msg("tryUpstream: serve raw domain with default branch")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ } else {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
+ http.StatusNotFound)
+ }
+}
diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go
new file mode 100644
index 0000000..df42d61
--- /dev/null
+++ b/server/handler/handler_sub_domain.go
@@ -0,0 +1,120 @@
+package handler
+
+import (
+ "fmt"
+ "net/http"
+ "path"
+ "strings"
+
+ "github.com/rs/zerolog"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/upstream"
+)
+
+func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix string,
+ trimmedHost string,
+ pathElements []string,
+ canonicalDomainCache cache.SetGetKey,
+) {
+ // Serve pages from subdomains of MainDomainSuffix
+ log.Debug().Msg("main domain suffix")
+
+ targetOwner := strings.TrimSuffix(trimmedHost, mainDomainSuffix)
+ targetRepo := pathElements[0]
+
+ if targetOwner == "www" {
+ // www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
+ ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect)
+ return
+ }
+
+ // Check if the first directory is a repo with the second directory as a branch
+ // example.codeberg.page/myrepo/@main/index.html
+ if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
+ if targetRepo == "pages" {
+ // example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
+ ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
+ return
+ }
+
+ log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: true,
+ TargetOwner: targetOwner,
+ TargetRepo: pathElements[0],
+ TargetBranch: pathElements[1][1:],
+ TargetPath: path.Join(pathElements[2:]...),
+ }, true); works {
+ log.Trace().Msg("tryUpstream: serve with specified repo and branch")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ } else {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
+ http.StatusFailedDependency)
+ }
+ return
+ }
+
+ // Check if the first directory is a branch for the "pages" repo
+ // example.codeberg.page/@main/index.html
+ if strings.HasPrefix(pathElements[0], "@") {
+ log.Debug().Msg("main domain preparations, now trying with specified branch")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: true,
+ TargetOwner: targetOwner,
+ TargetRepo: "pages",
+ TargetBranch: pathElements[0][1:],
+ TargetPath: path.Join(pathElements[1:]...),
+ }, true); works {
+ log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ } else {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
+ http.StatusFailedDependency)
+ }
+ return
+ }
+
+ // Check if the first directory is a repo with a "pages" branch
+ // example.codeberg.page/myrepo/index.html
+ // example.codeberg.page/pages/... is not allowed here.
+ log.Debug().Msg("main domain preparations, now trying with specified repo")
+ if pathElements[0] != "pages" {
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: true,
+ TargetOwner: targetOwner,
+ TargetRepo: pathElements[0],
+ TargetBranch: "pages",
+ TargetPath: path.Join(pathElements[1:]...),
+ }, false); works {
+ log.Debug().Msg("tryBranch, now trying upstream 5")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ return
+ }
+ }
+
+ // Try to use the "pages" repo on its default branch
+ // example.codeberg.page/index.html
+ log.Debug().Msg("main domain preparations, now trying with default repo/branch")
+ if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+ TryIndexPages: true,
+ TargetOwner: targetOwner,
+ TargetRepo: "pages",
+ TargetPath: path.Join(pathElements...),
+ }, false); works {
+ log.Debug().Msg("tryBranch, now trying upstream 6")
+ tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+ return
+ }
+
+ // Couldn't find a valid repo/branch
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("couldn't find a valid repo[%s]", targetRepo),
+ http.StatusFailedDependency)
+}
diff --git a/server/handler_test.go b/server/handler/handler_test.go
similarity index 85%
rename from server/handler_test.go
rename to server/handler/handler_test.go
index c0aca14..f5538c9 100644
--- a/server/handler_test.go
+++ b/server/handler/handler_test.go
@@ -1,4 +1,4 @@
-package server
+package handler
import (
"net/http/httptest"
@@ -11,12 +11,11 @@ import (
)
func TestHandlerPerformance(t *testing.T) {
- giteaRoot := "https://codeberg.org"
- giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false)
+ giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
testHandler := Handler(
"codeberg.page", "raw.codeberg.org",
giteaClient,
- giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
+ "https://docs.codeberg.org/pages/raw-content/",
[]string{"/.well-known/acme-challenge/"},
[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
cache.NewKeyValueCache(),
diff --git a/server/helpers.go b/server/handler/hsts.go
similarity index 96%
rename from server/helpers.go
rename to server/handler/hsts.go
index 7c898cd..1ab73ae 100644
--- a/server/helpers.go
+++ b/server/handler/hsts.go
@@ -1,4 +1,4 @@
-package server
+package handler
import (
"strings"
diff --git a/server/handler/try.go b/server/handler/try.go
new file mode 100644
index 0000000..b9adb7b
--- /dev/null
+++ b/server/handler/try.go
@@ -0,0 +1,76 @@
+package handler
+
+import (
+ "net/http"
+ "strings"
+
+ "github.com/rs/zerolog"
+
+ "codeberg.org/codeberg/pages/html"
+ "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/gitea"
+ "codeberg.org/codeberg/pages/server/upstream"
+)
+
+// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
+func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix, trimmedHost string,
+ options *upstream.Options,
+ canonicalDomainCache cache.SetGetKey,
+) {
+ // check if a canonical domain exists on a request on MainDomain
+ if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+ canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", string(mainDomainSuffix), canonicalDomainCache)
+ if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
+ canonicalPath := ctx.Req.RequestURI
+ if options.TargetRepo != "pages" {
+ path := strings.SplitN(canonicalPath, "/", 3)
+ if len(path) >= 3 {
+ canonicalPath = "/" + path[2]
+ }
+ }
+ ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
+ return
+ }
+ }
+
+ // add host for debugging
+ options.Host = string(trimmedHost)
+
+ // Try to request the file from the Gitea API
+ if !options.Upstream(ctx, giteaClient) {
+ html.ReturnErrorPage(ctx, "", ctx.StatusCode)
+ }
+}
+
+// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty,
+// it will also disallow search indexing and add a Link header to the canonical URL.
+func tryBranch(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+ targetOptions *upstream.Options, canonicalLink bool,
+) (*upstream.Options, bool) {
+ if targetOptions.TargetOwner == "" || targetOptions.TargetRepo == "" {
+ log.Debug().Msg("tryBranch: owner or repo is empty")
+ return nil, false
+ }
+
+ // Replace "~" to "/" so we can access branch that contains slash character
+ // Branch name cannot contain "~" so doing this is okay
+ targetOptions.TargetBranch = strings.ReplaceAll(targetOptions.TargetBranch, "~", "/")
+
+ // Check if the branch exists, otherwise treat it as a file path
+ branchExist, _ := targetOptions.GetBranchTimestamp(giteaClient)
+ if !branchExist {
+ log.Debug().Msg("tryBranch: branch doesn't exist")
+ return nil, false
+ }
+
+ if canonicalLink {
+ // Hide from search machines & add canonical link
+ ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex")
+ ctx.RespWriter.Header().Set("Link", targetOptions.ContentWebLink(giteaClient)+"; rel=\"canonical\"")
+ }
+
+ log.Debug().Msg("tryBranch: true")
+ return targetOptions, true
+}
diff --git a/server/try.go b/server/try.go
deleted file mode 100644
index 135c1e0..0000000
--- a/server/try.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package server
-
-import (
- "net/http"
- "strings"
-
- "codeberg.org/codeberg/pages/html"
- "codeberg.org/codeberg/pages/server/cache"
- "codeberg.org/codeberg/pages/server/context"
- "codeberg.org/codeberg/pages/server/gitea"
- "codeberg.org/codeberg/pages/server/upstream"
-)
-
-// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
- mainDomainSuffix, trimmedHost string,
-
- targetOptions *upstream.Options,
- targetOwner, targetRepo, targetBranch, targetPath string,
-
- canonicalDomainCache cache.SetGetKey,
-) {
- // check if a canonical domain exists on a request on MainDomain
- if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
- canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
- if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
- canonicalPath := ctx.Req.RequestURI
- if targetRepo != "pages" {
- path := strings.SplitN(canonicalPath, "/", 3)
- if len(path) >= 3 {
- canonicalPath = "/" + path[2]
- }
- }
- ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
- return
- }
- }
-
- targetOptions.TargetOwner = targetOwner
- targetOptions.TargetRepo = targetRepo
- targetOptions.TargetBranch = targetBranch
- targetOptions.TargetPath = targetPath
- targetOptions.Host = string(trimmedHost)
-
- // Try to request the file from the Gitea API
- if !targetOptions.Upstream(ctx, giteaClient) {
- html.ReturnErrorPage(ctx, "", ctx.StatusCode)
- }
-}
diff --git a/server/upstream/domains.go b/server/upstream/domains.go
index 6ad6506..0e29673 100644
--- a/server/upstream/domains.go
+++ b/server/upstream/domains.go
@@ -16,12 +16,12 @@ var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
domains []string
valid bool
)
- if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
+ if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
domains = cachedValue.([]string)
for _, domain := range domains {
if domain == actualDomain {
@@ -30,7 +30,7 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
}
}
} else {
- body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
+ body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
if err == nil {
for _, domain := range strings.Split(string(body), "\n") {
domain = strings.ToLower(domain)
@@ -45,16 +45,16 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
}
}
} else {
- log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo)
+ log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo)
}
- domains = append(domains, targetOwner+mainDomainSuffix)
+ domains = append(domains, o.TargetOwner+mainDomainSuffix)
if domains[len(domains)-1] == actualDomain {
valid = true
}
- if targetRepo != "" && targetRepo != "pages" {
- domains[len(domains)-1] += "/" + targetRepo
+ if o.TargetRepo != "" && o.TargetRepo != "pages" {
+ domains[len(domains)-1] += "/" + o.TargetRepo
}
- _ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
+ _ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout)
}
return domains[0], valid
}
diff --git a/server/upstream/header.go b/server/upstream/header.go
new file mode 100644
index 0000000..9575a3f
--- /dev/null
+++ b/server/upstream/header.go
@@ -0,0 +1,28 @@
+package upstream
+
+import (
+ "net/http"
+ "time"
+
+ "codeberg.org/codeberg/pages/server/context"
+ "codeberg.org/codeberg/pages/server/gitea"
+)
+
+// setHeader set values to response header
+func (o *Options) setHeader(ctx *context.Context, header http.Header) {
+ if eTag := header.Get(gitea.ETagHeader); eTag != "" {
+ ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
+ }
+ if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
+ ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
+ }
+ if length := header.Get(gitea.ContentLengthHeader); length != "" {
+ ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
+ }
+ if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
+ } else {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
+ }
+ ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
+}
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
index 6bc23c8..428976b 100644
--- a/server/upstream/helper.go
+++ b/server/upstream/helper.go
@@ -2,35 +2,46 @@ package upstream
import (
"errors"
+ "fmt"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/gitea"
)
-// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
-// (or nil if the branch doesn't exist)
-func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp {
- log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
+// GetBranchTimestamp finds the default branch (if branch is "") and save branch and it's last modification time to Options
+func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) {
+ log := log.With().Strs("BranchInfo", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch}).Logger()
- if len(branch) == 0 {
+ if len(o.TargetBranch) == 0 {
// Get default branch
- defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
+ defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(o.TargetOwner, o.TargetRepo)
if err != nil {
log.Err(err).Msg("Could't fetch default branch from repository")
- return nil
+ return false, err
}
log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
- branch = defaultBranch
+ o.TargetBranch = defaultBranch
}
- timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch)
+ timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch)
if err != nil {
if !errors.Is(err, gitea.ErrorNotFound) {
log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch")
}
- return nil
+ return false, err
}
+
+ if timestamp == nil || timestamp.Branch == "" {
+ return false, fmt.Errorf("empty response")
+ }
+
log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp)
- return timestamp
+ o.BranchTimestamp = timestamp.Timestamp
+ o.TargetBranch = timestamp.Branch
+ return true, nil
+}
+
+func (o *Options) ContentWebLink(giteaClient *gitea.Client) string {
+ return giteaClient.ContentWebLink(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath) + "; rel=\"canonical\""
}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index d37c35e..b76b8e6 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -34,10 +34,10 @@ var upstreamNotFoundPages = []string{
// Options provides various options for the upstream request.
type Options struct {
- TargetOwner,
- TargetRepo,
- TargetBranch,
- TargetPath,
+ TargetOwner string
+ TargetRepo string
+ TargetBranch string
+ TargetPath string
// Used for debugging purposes.
Host string
@@ -62,16 +62,22 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
- branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
-
- if branch == nil || branch.Branch == "" {
+ branchExist, err := o.GetBranchTimestamp(giteaClient)
+ // handle 404
+ if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist {
html.ReturnErrorPage(ctx,
- fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
+ fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
+ http.StatusNotFound)
+ return true
+ }
+
+ // handle unexpected errors
+ if err != nil {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err),
http.StatusFailedDependency)
return true
}
- o.TargetBranch = branch.Branch
- o.BranchTimestamp = branch.Timestamp
}
// Check if the browser has a cached version
@@ -172,21 +178,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
}
// Set ETag & MIME
- if eTag := header.Get(gitea.ETagHeader); eTag != "" {
- ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
- }
- if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
- ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
- }
- if length := header.Get(gitea.ContentLengthHeader); length != "" {
- ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
- }
- if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
- ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
- } else {
- ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
- }
- ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
+ o.setHeader(ctx, header)
log.Debug().Msg("Prepare response")