Upstream
/
pages-server
mirror of https://codeberg.org/Codeberg/pages-server.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

WIP

This commit is contained in:
crapStone 2023-11-17 16:15:25 +01:00 committed by crapStone
parent dd5124912e
commit 0e334d8e64
32 changed files with 611 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Justfile
View File

@ -9,7 +9,7 @@ dev:
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
export HTTP_PORT=8880
export HTTP_PORT=1234
export ENABLE_HTTP_SERVER=true
export LOG_LEVEL=trace
go run -tags '{{TAGS}}' .
@ -42,10 +42,10 @@ tool-gofumpt:
fi
test:
go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
test-run TEST:
go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
integration:
go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/...

6
cmd/certs.go → cli/certs.go
View File

@ -1,4 +1,4 @@
package cmd
package cli
import (
"fmt"
@ -26,7 +26,7 @@ var Certs = &cli.Command{
}
func listCerts(ctx *cli.Context) error {
certDB, closeFn, err := openCertDB(ctx)
certDB, closeFn, err := OpenCertDB(ctx)
if err != nil {
return err
}
@ -53,7 +53,7 @@ func removeCert(ctx *cli.Context) error {
domains := ctx.Args().Slice()
certDB, closeFn, err := openCertDB(ctx)
certDB, closeFn, err := OpenCertDB(ctx)
if err != nil {
return err
}

33
cmd/flags.go → cli/flags.go
View File

@ -1,4 +1,4 @@
package cmd
package cli
import (
"github.com/urfave/cli/v2"
@ -98,12 +98,7 @@ var (
Name: "enable-http-server",
Usage: "start a http server to redirect to https and respond to http acme challenges",
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
&cli.StringFlag{
Name: "log-level",
Value: "warn",
Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal",
EnvVars: []string{"LOG_LEVEL"},
Value: false,
},
// Default branches to fetch assets from
&cli.StringSliceFlag{
@ -113,6 +108,30 @@ var (
Value: cli.NewStringSlice("pages"),
},
&cli.StringSliceFlag{
Name: "allowed-cors-domains",
Usage: "specify allowed CORS domains",
EnvVars: []string{"ALLOWED_CORS_DOMAINS"},
},
&cli.StringSliceFlag{
Name: "blacklisted-paths",
Usage: "return an error on these url paths",
EnvVars: []string{"BLACKLISTED_PATHS"},
},
&cli.StringFlag{
Name: "log-level",
Value: "warn",
Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal",
EnvVars: []string{"LOG_LEVEL"},
},
&cli.StringFlag{
Name: "config-file",
Usage: "specify the location of the config file",
Aliases: []string{"config"},
EnvVars: []string{"CONFIG_FILE"},
},
// ############################
// ### ACME Client Settings ###
// ############################

20
cmd/setup.go → cli/setup.go
View File

@ -1,4 +1,4 @@
package cmd
package cli
import (
"errors"
@ -10,11 +10,25 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/version"
)
var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
func CreatePagesApp() *cli.App {
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version.Version
app.Usage = "pages server"
app.Flags = ServerFlags
app.Commands = []*cli.Command{
Certs,
}
return app
}
func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err)
@ -29,7 +43,7 @@ func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err e
return certDB, closeFn, nil
}
func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) {
func CreateAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) {
acmeAPI := ctx.String("acme-api-endpoint")
acmeMail := ctx.String("acme-email")
acmeEabHmac := ctx.String("acme-eab-hmac")

150
cmd/main.go
View File

@ -1,150 +0,0 @@
package cmd
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/handler"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
// TODO: make it a flag
var AllowedCorsDomains = []string{
"fonts.codeberg.org",
"design.codeberg.org",
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
// TODO: Make it a flag too
var BlacklistedPaths = []string{
"/.well-known/acme-challenge/",
}
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
// Initialize the logger.
logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
if err != nil {
return err
}
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
giteaRoot := ctx.String("gitea-root")
giteaAPIToken := ctx.String("gitea-api-token")
rawDomain := ctx.String("raw-domain")
defaultBranches := ctx.StringSlice("pages-branch")
mainDomainSuffix := ctx.String("pages-domain")
listeningHost := ctx.String("host")
listeningSSLPort := ctx.Uint("port")
listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
enableHTTPServer := ctx.Bool("enable-http-server")
allowedCorsDomains := AllowedCorsDomains
if rawDomain != "" {
allowedCorsDomains = append(allowedCorsDomains, rawDomain)
}
// Make sure MainDomain has a trailing dot
if !strings.HasPrefix(mainDomainSuffix, ".") {
mainDomainSuffix = "." + mainDomainSuffix
}
if len(defaultBranches) == 0 {
return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
}
// Init ssl cert database
certDB, closeFn, err := openCertDB(ctx)
if err != nil {
return err
}
defer closeFn()
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewKeyValueCache()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewKeyValueCache()
// clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewKeyValueCache()
giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache)
if err != nil {
return err
}
if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil {
return err
}
// Create listener for SSL connections
log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
acmeClient,
defaultBranches[0],
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
if enableHTTPServer {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
// Create listener for http and start listening
go func() {
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache, redirectsCache)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
if err := http.Serve(listener, sslHandler); err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")
}
return nil
}

31
config/assets/test_config.toml Normal file
View File

@ -0,0 +1,31 @@
logLevel = 'trace'
[server]
host = '127.0.0.1'
port = 443
httpPort = 80
httpServerEnabled = true
mainDomain = 'codeberg.page'
rawDomain = 'raw.codeberg.page'
allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org']
blacklistedPaths = []
[gitea]
root = 'codeberg.org'
token = 'XXXXX'
lfsEnabled = true
followSymlinks = true
[database]
type = 'sqlite'
conn = 'certs.sqlite'
[ACME]
email = 'a@b.c'
apiEndpoint = 'https://example.com'
acceptTerms = false
useRateLimits = true
eab_hmac = ''
eab_kid = ''
dnsProvider = ''
accountConfigFile = ''

43
config/config.go Normal file
View File

@ -0,0 +1,43 @@
package config
type Config struct {
LogLevel string
Server ServerConfig
Gitea GiteaConfig
Database DatabaseConfig
ACME ACMEConfig
}
type ServerConfig struct {
Host string
Port uint16
HttpPort uint16
HttpServerEnabled bool
MainDomain string
RawDomain string
AllowedCorsDomains []string
BlacklistedPaths []string
}
type GiteaConfig struct {
Root string
Token string
LFSEnabled bool
FollowSymlinks bool
}
type DatabaseConfig struct {
Type string
Conn string
}
type ACMEConfig struct {
Email string
APIEndpoint string
AcceptTerms bool
UseRateLimits bool
EAB_HMAC string
EAB_KID string
DNSProvider string
AccountConfigFile string
}

32
config/program.go Normal file
View File

@ -0,0 +1,32 @@
package config
import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/gitea"
)
type HandlerConfig struct {
mainDomainSuffix string
rawDomain string
giteaClient *gitea.Client
blacklistedPaths []string
allowedCorsDomains []string
defaultBranches []string
dnsLookupCache cache.ICache
canonicalDomainCache cache.ICache
redirectsCache cache.ICache
}
type TLSConfig struct {
mainDomainSuffix string
firstDefaultBranch string
giteaClient *gitea.Client
acmeClient *certificates.AcmeClient
keyCache cache.ICache
challengeCache cache.ICache
dnsLookupCache cache.ICache
canonicalDomainCache cache.ICache
certDB database.CertDB
}

77
config/setup.go Normal file
View File

@ -0,0 +1,77 @@
package config
import (
"os"
"path"
"github.com/pelletier/go-toml/v2"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)
var ALWAYS_BLACKLISTED_PATHS = []string{
"/.well-known/acme-challenge/",
}
func ReadConfig(ctx *cli.Context) (*Config, error) {
// if config is not given as argument return empty config
if !ctx.IsSet("config-file") {
return &Config{}, nil
}
configFile := path.Clean(ctx.String("config-file"))
log.Debug().Str("config-file", configFile).Msg("reading config file")
content, err := os.ReadFile(configFile)
if err != nil {
return nil, err
}
config := &Config{}
err = toml.Unmarshal(content, config)
return config, err
}
func MergeConfig(ctx *cli.Context, config *Config) {
if ctx.IsSet("log-level") {
config.LogLevel = ctx.String("log-level")
}
mergeServerConfig(ctx, &config.Server)
}
func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
if ctx.IsSet("host") {
config.Host = ctx.String("host")
}
if ctx.IsSet("port") {
config.Port = uint16(ctx.Uint("port"))
}
if ctx.IsSet("http-port") {
config.HttpPort = uint16(ctx.Uint("http-port"))
}
if ctx.IsSet("enable-http-server") {
config.HttpServerEnabled = ctx.Bool("enable-http-server")
}
if ctx.IsSet("pages-domain") {
config.MainDomain = ctx.String("pages-domain")
}
if ctx.IsSet("raw-domain") {
config.RawDomain = ctx.String("raw-domain")
}
if ctx.IsSet("allowed-cors-domains") {
config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains")
}
if ctx.IsSet("blacklisted-paths") {
config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths")
}
// add the paths that should always be blacklisted
config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
}
func mergeGiteaConfig(ctx *cli.Context, config *GiteaConfig) {
if ctx.IsSet("gitea-root") {
config.Root = ctx.String("gitea-root")
}
}

167
config/setup_test.go Normal file
View File

@ -0,0 +1,167 @@
package config
import (
"os"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert"
"github.com/urfave/cli/v2"
cmd "codeberg.org/codeberg/pages/cli"
)
func runApp(t *testing.T, fn func(*cli.Context) error, args []string) {
app := cmd.CreatePagesApp()
app.Action = fn
// os.Args always contains the binary name
args = append([]string{"testing"}, args...)
err := app.Run(args)
assert.NoError(t, err)
}
func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg, err := ReadConfig(ctx)
assert.Equal(t, &Config{}, cfg)
return err
},
[]string{},
)
}
func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
content, err := os.ReadFile("assets/test_config.toml")
if err != nil {
return err
}
expectedConfig := &Config{}
err = toml.Unmarshal(content, expectedConfig)
if err != nil {
return err
}
cfg, err := ReadConfig(ctx)
assert.Equal(t, expectedConfig, cfg)
return err
},
[]string{"--config-file", "assets/test_config.toml"},
)
}
func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &ServerConfig{}
mergeServerConfig(ctx, cfg)
expected := ALWAYS_BLACKLISTED_PATHS
assert.Equal(t, expected, cfg.BlacklistedPaths)
return nil
},
[]string{},
)
}
func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &ServerConfig{
Host: "original",
Port: 8080,
HttpPort: 80,
HttpServerEnabled: false,
MainDomain: "original",
RawDomain: "original",
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
}
mergeServerConfig(ctx, cfg)
expectedConfig := &ServerConfig{
Host: "changed",
Port: 8443,
HttpPort: 443,
HttpServerEnabled: true,
MainDomain: "changed",
RawDomain: "changed",
AllowedCorsDomains: []string{"changed"},
BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...),
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--pages-domain", "changed",
"--raw-domain", "changed",
"--allowed-cors-domains", "changed",
"--blacklisted-paths", "changed",
"--host", "changed",
"--port", "8443",
"--http-port", "443",
"--enable-http-server",
},
)
}
func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
type testValuePair struct {
args []string
callback func(*ServerConfig)
}
testValuePairs := []testValuePair{
{args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }},
{args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }},
{args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }},
{args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }},
{args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }},
{args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }},
{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed", "changed"} }}, // don't ask why, the cli lib always adds two strings when running all tests and one when running a single test
{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed", "changed"} }}, // same here
}
for _, pair := range testValuePairs {
runApp(
t,
func(ctx *cli.Context) error {
cfg := ServerConfig{
Host: "original",
Port: 8080,
HttpPort: 80,
HttpServerEnabled: false,
MainDomain: "original",
RawDomain: "original",
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
}
expectedConfig := cfg
pair.callback(&expectedConfig)
expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
mergeServerConfig(ctx, &cfg)
assert.Equal(t, expectedConfig, cfg)
return nil
},
pair.args,
)
}
}

31
example_config.toml Normal file
View File

@ -0,0 +1,31 @@
logLevel = 'debug'
[server]
host = '127.0.0.1'
port = 443
httpPort = 80
httpServerEnabled = false
mainDomain = ''
rawDomain = ''
allowedCorsDomains = ['asdf', 'jkl;']
blacklistedPaths = []
[gitea]
root = ''
token = ''
lfsEnabled = false
followSymlinks = false
[database]
type = 'sqlite'
conn = 'certs.sqlite'
[ACME]
email = ''
apiEndpoint = ''
acceptTerms = false
useRateLimits = false
eab_hmac = ''
eab_kid = ''
dnsProvider = ''
accountConfigFile = ''

7
go.mod
View File

@ -13,9 +13,10 @@ require (
github.com/lib/pq v1.10.7
github.com/mattn/go-sqlite3 v1.14.16
github.com/microcosm-cc/bluemonday v1.0.26
github.com/pelletier/go-toml/v2 v2.1.0
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
github.com/rs/zerolog v1.27.0
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.3.0
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
xorm.io/xorm v1.3.2
@ -113,7 +114,7 @@ require (
github.com/softlayer/softlayer-go v1.0.3 // indirect
github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/transip/gotransip/v6 v6.6.1 // indirect
github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
@ -135,6 +136,6 @@ require (
gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
xorm.io/builder v0.3.12 // indirect
)

14
go.sum
View File

@ -570,6 +570,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@ -680,14 +682,19 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@ -1079,8 +1086,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

2
integration/main_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"time"
"codeberg.org/codeberg/pages/cmd"
cmd "codeberg.org/codeberg/pages/cli"
"github.com/urfave/cli/v2"
)

151
main.go
View File

@ -1,29 +1,156 @@
package main
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
_ "github.com/joho/godotenv/autoload"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/cmd"
"codeberg.org/codeberg/pages/server/version"
cmd "codeberg.org/codeberg/pages/cli"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/handler"
)
func main() {
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version.Version
app.Usage = "pages server"
app.Action = cmd.Serve
app.Flags = cmd.ServerFlags
app.Commands = []*cli.Command{
cmd.Certs,
}
app := cmd.CreatePagesApp()
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
log.Error().Err(err).Msg("A fatal error occurred")
os.Exit(1)
}
}
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
// initialize logger with Trace, overridden later with actual level
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel)
cfg, err := config.ReadConfig(ctx)
if err != nil {
log.Error().Err(err).Msg("could not read config")
}
config.MergeConfig(ctx, cfg)
// Initialize the logger.
logLevel, err := zerolog.ParseLevel(cfg.LogLevel)
if err != nil {
return err
}
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
giteaRoot := ctx.String("gitea-root")
giteaAPIToken := ctx.String("gitea-api-token")
rawDomain := ctx.String("raw-domain")
defaultBranches := ctx.StringSlice("pages-branch")
mainDomainSuffix := ctx.String("pages-domain")
listeningHost := ctx.String("host")
listeningSSLPort := ctx.Uint("port")
listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
enableHTTPServer := ctx.Bool("enable-http-server")
allowedCorsDomains := cfg.Server.AllowedCorsDomains
if rawDomain != "" {
allowedCorsDomains = append(allowedCorsDomains, rawDomain)
}
// Make sure MainDomain has a trailing dot
if !strings.HasPrefix(mainDomainSuffix, ".") {
mainDomainSuffix = "." + mainDomainSuffix
}
if len(defaultBranches) == 0 {
return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
}
// Init ssl cert database
certDB, closeFn, err := cmd.OpenCertDB(ctx)
if err != nil {
return err
}
defer closeFn()
keyCache := cache.NewInMemoryCache()
challengeCache := cache.NewInMemoryCache()
// canonicalDomainCache stores canonical domains
canonicalDomainCache := cache.NewInMemoryCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewInMemoryCache()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewInMemoryCache()
// clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewInMemoryCache()
giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
acmeClient, err := cmd.CreateAcmeClient(ctx, enableHTTPServer, challengeCache)
if err != nil {
return err
}
if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil {
return err
}
// Create listener for SSL connections
log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
acmeClient,
defaultBranches[0],
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
if enableHTTPServer {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
// Create listener for http and start listening
go func() {
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP server")
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
cfg.Server.BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache, redirectsCache)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
return http.Serve(listener, sslHandler)
}

2
server/cache/interface.go vendored
View File

@ -2,7 +2,7 @@ package cache
import "time"
type SetGetKey interface {
type ICache interface {
Set(key string, value interface{}, ttl time.Duration) error
Get(key string) (interface{}, bool)
Remove(key string)

2
server/cache/setup.go → server/cache/memory.go vendored
View File

@ -2,6 +2,6 @@ package cache
import "github.com/OrlovEvgeny/go-mcache"
func NewKeyValueCache() SetGetKey {
func NewInMemoryCache() ICache {
return mcache.New()
}

2
server/certificates/acme_client.go
View File

@ -28,7 +28,7 @@ type AcmeClient struct {
acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
}
func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) {
func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.ICache) (*AcmeClient, error) {
acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
if err != nil {
return nil, err

6
server/certificates/cached_challengers.go
View File

@ -15,7 +15,7 @@ import (
)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
challengeCache cache.ICache
}
// make sure AcmeTLSChallengeProvider match Provider interface
@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
challengeCache cache.ICache
}
// make sure AcmeHTTPChallengeProvider match Provider interface
@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
return nil
}
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc {
// handle custom-ssl-ports to be added on https redirects
portPart := ""
if sslPort != 443 {

2
server/certificates/certificates.go
View File

@ -31,7 +31,7 @@ func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
acmeClient *AcmeClient,
firstDefaultBranch string,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache,
certDB database.CertDB,
) *tls.Config {
return &tls.Config{

2
server/dns/dns.go
View File

@ -15,7 +15,7 @@ var defaultPagesRepo = "pages"
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error

4
server/gitea/cache.go
View File

@ -74,7 +74,7 @@ type writeCacheReader struct {
buffer *bytes.Buffer
rileResponse *FileResponse
cacheKey string
cache cache.SetGetKey
cache cache.ICache
hasError bool
}
@ -99,7 +99,7 @@ func (t *writeCacheReader) Close() error {
return t.originalReader.Close()
}
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser {
if r == nil || cache == nil || cacheKey == "" {
log.Error().Msg("could not create CacheReader")
return nil

4
server/gitea/client.go
View File

@ -44,7 +44,7 @@ const (
type Client struct {
sdkClient *gitea.Client
responseCache cache.SetGetKey
responseCache cache.ICache
giteaRoot string
@ -55,7 +55,7 @@ type Client struct {
defaultMimeType string
}
func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
func NewClient(giteaRoot, giteaAPIToken string, respCache cache.ICache, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
if err != nil {
return nil, err

2
server/handler/handler.go
View File

@ -23,7 +23,7 @@ func Handler(mainDomainSuffix, rawDomain string,
giteaClient *gitea.Client,
blacklistedPaths, allowedCorsDomains []string,
defaultPagesBranches []string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()

2
server/handler/handler_custom_domain.go
View File

@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
trimmedHost string,
pathElements []string,
firstDefaultBranch string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)

2
server/handler/handler_raw_domain.go
View File

@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
mainDomainSuffix string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")

2
server/handler/handler_sub_domain.go
View File

@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
defaultPagesBranches []string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")

8
server/handler/handler_test.go
View File

@ -12,16 +12,16 @@ import (
)
func TestHandlerPerformance(t *testing.T) {
giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewInMemoryCache(), false, false)
testHandler := Handler(
"codeberg.page", "raw.codeberg.org",
giteaClient,
[]string{"/.well-known/acme-challenge/"},
[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
[]string{"pages"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewInMemoryCache(),
cache.NewInMemoryCache(),
cache.NewInMemoryCache(),
)
testCase := func(uri string, status int) {

4
server/handler/try.go
View File

@ -17,8 +17,8 @@ import (
func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost string,
options *upstream.Options,
canonicalDomainCache cache.SetGetKey,
redirectsCache cache.SetGetKey,
canonicalDomainCache cache.ICache,
redirectsCache cache.ICache,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {

2
server/upstream/domains.go
View File

@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) {
func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
// Check if this request is cached.
if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
domains := cachedValue.([]string)

4
server/upstream/redirects.go
View File

@ -23,7 +23,7 @@ var redirectsCacheTimeout = 10 * time.Minute
const redirectsConfig = "_redirects"
// getRedirects returns redirects specified in the _redirects file.
func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect {
func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect {
var redirects []Redirect
cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
@ -63,7 +63,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S
return redirects
}
func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) {
func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) {
if len(redirects) > 0 {
for _, redirect := range redirects {
reqUrl := ctx.Req.RequestURI

2
server/upstream/upstream.go
View File

@ -53,7 +53,7 @@ type Options struct {
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool {
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.TargetOwner == "" || o.TargetRepo == "" {