Merge remote-tracking branch 'upstream/main' into issue115

This commit is contained in:
6543 2023-02-14 03:16:52 +01:00
commit c5040b622d
No known key found for this signature in database
GPG Key ID: B8BE6D610E61C862
15 changed files with 241 additions and 115 deletions

1
.gitignore vendored
View File

@ -7,3 +7,4 @@ build/
vendor/
pages
certs.sqlite
.bash_history

View File

@ -50,3 +50,6 @@ integration:
integration-run TEST:
go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
docker:
docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just

View File

@ -8,11 +8,13 @@ var (
CertStorageFlags = []cli.Flag{
&cli.StringFlag{
Name: "db-type",
Usage: "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io",
Value: "sqlite3",
EnvVars: []string{"DB_TYPE"},
},
&cli.StringFlag{
Name: "db-conn",
Usage: "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access",
Value: "certs.sqlite",
EnvVars: []string{"DB_CONN"},
},
@ -87,15 +89,21 @@ var (
EnvVars: []string{"HOST"},
Value: "[::]",
},
&cli.StringFlag{
&cli.UintFlag{
Name: "port",
Usage: "specifies port of listening address",
EnvVars: []string{"PORT"},
Value: "443",
Usage: "specifies the https port to listen to ssl requests",
EnvVars: []string{"PORT", "HTTPS_PORT"},
Value: 443,
},
&cli.UintFlag{
Name: "http-port",
Usage: "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true",
EnvVars: []string{"HTTP_PORT"},
Value: 80,
},
&cli.BoolFlag{
Name: "enable-http-server",
// TODO: desc
Name: "enable-http-server",
Usage: "start a http server to redirect to https and respond to http acme challenges",
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
&cli.StringFlag{
@ -132,23 +140,23 @@ var (
Value: true,
},
&cli.BoolFlag{
Name: "acme-accept-terms",
// TODO: Usage
Name: "acme-accept-terms",
Usage: "To accept the ACME ToS",
EnvVars: []string{"ACME_ACCEPT_TERMS"},
},
&cli.StringFlag{
Name: "acme-eab-kid",
// TODO: Usage
Name: "acme-eab-kid",
Usage: "Register the current account to the ACME server with external binding.",
EnvVars: []string{"ACME_EAB_KID"},
},
&cli.StringFlag{
Name: "acme-eab-hmac",
// TODO: Usage
Name: "acme-eab-hmac",
Usage: "Register the current account to the ACME server with external binding.",
EnvVars: []string{"ACME_EAB_HMAC"},
},
&cli.StringFlag{
Name: "dns-provider",
Usage: "Use DNS-Challenge for main domain\n\nRead more at: https://go-acme.github.io/lego/dns/",
Usage: "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/",
EnvVars: []string{"DNS_PROVIDER"},
},
&cli.StringFlag{

View File

@ -14,7 +14,6 @@ import (
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
@ -49,7 +48,9 @@ func Serve(ctx *cli.Context) error {
defaultBranches := ctx.StringSlice("pages-branch")
mainDomainSuffix := ctx.String("pages-domain")
rawInfoPage := ctx.String("raw-info-page")
listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
listeningHost := ctx.String("host")
listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("port"))
listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
enableHTTPServer := ctx.Bool("enable-http-server")
allowedCorsDomains := AllowedCorsDomains
@ -96,23 +97,14 @@ func Serve(ctx *cli.Context) error {
return err
}
// Create handler based on settings
httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache)
httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
// Setup listener and TLS
log.Info().Msgf("Listening on https://%s", listeningAddress)
listener, err := net.Listen("tcp", listeningAddress)
// Create listener for SSL connections
log.Info().Msgf("Listening on https://%s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
acmeClient,
@ -126,18 +118,30 @@ func Serve(ctx *cli.Context) error {
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
if enableHTTPServer {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache)
// Create listener for http and start listening
go func() {
log.Info().Msg("Start HTTP server listening on :80")
err := http.ListenAndServe("[::]:80", httpHandler)
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
}
}()
}
// Start the web fastServer
// Create ssl handler based on settings
sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache)
// Start the ssl listener
log.Info().Msgf("Start listening on %s", listener.Addr())
if err := http.Serve(listener, httpsHandler); err != nil {
if err := http.Serve(listener, sslHandler); err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")
}

View File

@ -43,6 +43,11 @@ func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache ca
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
}
if acmeEabHmac != "" && acmeEabKID == "" {
return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
} else if acmeEabHmac == "" && acmeEabKID != "" {
return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
}
return certificates.NewAcmeClient(
acmeAccountConf,

View File

@ -109,6 +109,34 @@ func TestCustomDomainRedirects(t *testing.T) {
assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location"))
}
func TestRawCustomDomain(t *testing.T) {
log.Println("=== TestRawCustomDomain ===")
// test raw domain response for custom domain branch
resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "76", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 76, getSize(resp.Body))
}
func TestRawIndex(t *testing.T) {
log.Println("=== TestRawCustomDomain ===")
// test raw domain response for index.html
resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "597", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 597, getSize(resp.Body))
}
func TestGetNotFound(t *testing.T) {
log.Println("=== TestGetNotFound ===")
// test custom not found pages

View File

@ -14,6 +14,8 @@ import (
"github.com/rs/zerolog/log"
)
const challengePath = "/.well-known/acme-challenge/"
func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config

View File

@ -1,11 +1,15 @@
package certificates
import (
"net/http"
"strings"
"time"
"github.com/go-acme/lego/v4/challenge"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/utils"
)
type AcmeTLSChallengeProvider struct {
@ -39,3 +43,18 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
}
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := context.New(w, req)
if strings.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
if !ok || challenge == nil {
ctx.String("no challenge for this token", http.StatusNotFound)
}
ctx.String(challenge.(string))
} else {
ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
}
}
}

View File

@ -37,22 +37,23 @@ func TLSConfig(mainDomainSuffix string,
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
if len(sni) < 1 {
return nil, errors.New("missing sni")
domain := strings.ToLower(strings.TrimSpace(info.ServerName))
if len(domain) < 1 {
return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)")
}
// https request init is actually a acme challenge
if info.SupportedProtos != nil {
for _, proto := range info.SupportedProtos {
if proto != tlsalpn01.ACMETLS1Protocol {
continue
}
challenge, ok := challengeCache.Get(sni)
challenge, ok := challengeCache.Get(domain)
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
if err != nil {
return nil, err
}
@ -62,22 +63,22 @@ func TLSConfig(mainDomainSuffix string,
targetOwner := ""
mayObtainCert := true
if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
sni = mainDomainSuffix
domain = mainDomainSuffix
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
sni = mainDomainSuffix
domain = mainDomainSuffix
} else {
targetOpt := &upstream.Options{
TargetOwner: targetOwner,
TargetRepo: targetRepo,
TargetBranch: targetBranch,
}
_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
_, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache)
if !valid {
// We shouldn't obtain a certificate when we cannot check if the
// repository has specified this domain in the `.domains` file.
@ -86,30 +87,34 @@ func TLSConfig(mainDomainSuffix string,
}
}
if tlsCertificate, ok := keyCache.Get(sni); ok {
if tlsCertificate, ok := keyCache.Get(domain); ok {
// we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil
}
var tlsCertificate *tls.Certificate
var err error
if tlsCertificate, err = acmeClient.retrieveCertFromDB(sni, mainDomainSuffix, false, certDB); err != nil {
// request a new certificate
if strings.EqualFold(sni, mainDomainSuffix) {
if tlsCertificate, err = acmeClient.retrieveCertFromDB(domain, mainDomainSuffix, false, certDB); err != nil {
if !errors.Is(err, database.ErrNotFound) {
return nil, err
}
// we could not find a cert in db, request a new certificate
// first check if we are allowed to obtain a cert for this domain
if strings.EqualFold(domain, mainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
if !mayObtainCert {
return nil, fmt.Errorf("won't request certificate for %q", sni)
return nil, fmt.Errorf("won't request certificate for %q", domain)
}
tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{sni}, nil, targetOwner, false, mainDomainSuffix, certDB)
tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB)
if err != nil {
return nil, err
}
}
if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil {
if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil {
return nil, err
}
return tlsCertificate, nil
@ -165,7 +170,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
if !strings.EqualFold(sni, mainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err)
return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err)
}
// renew certificates 7 days before they expire

View File

@ -4,13 +4,15 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"codeberg.org/codeberg/pages/server/database"
)
func TestMockCert(t *testing.T) {
db, err := database.NewTmpDB()
assert.NoError(t, err)
db := database.NewMockCertDB(t)
db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil)
cert, err := mockCert("example.com", "some error msg", "codeberg.page", db)
assert.NoError(t, err)
if assert.NotEmpty(t, cert) {

View File

@ -8,6 +8,9 @@ import (
"github.com/rs/zerolog/log"
)
//go:generate go install github.com/vektra/mockery/v2@latest
//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore
type CertDB interface {
Close() error
Put(name string, cert *certificate.Resource) error

View File

@ -1,49 +1,122 @@
// Code generated by mockery v2.20.0. DO NOT EDIT.
package database
import (
"fmt"
"time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/go-acme/lego/v4/certificate"
certificate "github.com/go-acme/lego/v4/certificate"
mock "github.com/stretchr/testify/mock"
)
var _ CertDB = tmpDB{}
type tmpDB struct {
intern *mcache.CacheDriver
ttl time.Duration
// MockCertDB is an autogenerated mock type for the CertDB type
type MockCertDB struct {
mock.Mock
}
func (p tmpDB) Close() error {
_ = p.intern.Close()
return nil
}
// Close provides a mock function with given fields:
func (_m *MockCertDB) Close() error {
ret := _m.Called()
func (p tmpDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Set(name, cert, p.ttl)
}
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
return nil, fmt.Errorf("cert for %q not found", name)
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return cert.(*certificate.Resource), nil
return r0
}
func (p tmpDB) Delete(key string) error {
p.intern.Remove(key)
return nil
// Delete provides a mock function with given fields: key
func (_m *MockCertDB) Delete(key string) error {
ret := _m.Called(key)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(key)
} else {
r0 = ret.Error(0)
}
return r0
}
func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
return nil, fmt.Errorf("items not implemented for tmpDB")
// Get provides a mock function with given fields: name
func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) {
ret := _m.Called(name)
var r0 *certificate.Resource
var r1 error
if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok {
return rf(name)
}
if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*certificate.Resource)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
func NewTmpDB() (CertDB, error) {
return &tmpDB{
intern: mcache.New(),
ttl: time.Minute,
}, nil
// Items provides a mock function with given fields: page, pageSize
func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) {
ret := _m.Called(page, pageSize)
var r0 []*Cert
var r1 error
if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok {
return rf(page, pageSize)
}
if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok {
r0 = rf(page, pageSize)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*Cert)
}
}
if rf, ok := ret.Get(1).(func(int, int) error); ok {
r1 = rf(page, pageSize)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Put provides a mock function with given fields: name, cert
func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error {
ret := _m.Called(name, cert)
var r0 error
if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok {
r0 = rf(name, cert)
} else {
r0 = ret.Error(0)
}
return r0
}
type mockConstructorTestingTNewMockCertDB interface {
mock.TestingT
Cleanup(func())
}
// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB {
mock := &MockCertDB{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@ -20,7 +20,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
canonicalDomainCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
canonicalPath := ctx.Req.RequestURI

View File

@ -1,27 +0,0 @@
package server
import (
"net/http"
"strings"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/utils"
)
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
challengePath := "/.well-known/acme-challenge/"
return func(w http.ResponseWriter, req *http.Request) {
ctx := context.New(w, req)
if strings.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
if !ok || challenge == nil {
ctx.String("no challenge for this token", http.StatusNotFound)
}
ctx.String(challenge.(string))
} else {
ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
}
}
}

View File

@ -168,7 +168,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
return true
}
if strings.HasSuffix(ctx.Path(), "/index.html") {
if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
return true
}