Upstream
/
pages-server
mirror of https://codeberg.org/Codeberg/pages-server.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

wip

This commit is contained in:
6543 2023-02-09 19:14:53 +01:00
parent 75942990ac
commit 3c0ee7e8a3
9 changed files with 192 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ acme-account.json
build/
vendor/
pages
certs.sqlite

1
Dockerfile
View File

@ -2,7 +2,6 @@ FROM techknowlogick/xgo as build
WORKDIR /workspace
RUN apk add ca-certificates
COPY . .
RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' .

17
Justfile
View File

@ -1,3 +1,6 @@
CGO_FLAGS := '-extldflags "-static" -linkmode external'
TAGS := 'sqlite sqlite_unlock_notify netgo'
dev:
#!/usr/bin/env bash
set -euxo pipefail
@ -7,13 +10,13 @@ dev:
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
export LOG_LEVEL=trace
go run -tags 'sqlite sqlite_unlock_notify -ldflags '-s -w -extldflags "-static" -linkmode external' netgo' .
go run -tags 'sqlite sqlite_unlock_notify netgo' .
build:
CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./
CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./
build-tag VERSION:
CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./
CGO_ENABLED=1 go build -tags '{{TAGS}}' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./
lint: tool-golangci tool-gofumpt
[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
@ -38,13 +41,13 @@ tool-gofumpt:
fi
test:
go test -race -tags 'sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
test-run TEST:
go test -race -tags 'sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
integration:
go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/integration/...
go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/...
integration-run TEST:
go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

118
cmd/certs.go
View File

@ -4,6 +4,8 @@ import (
"fmt"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/database"
@ -23,6 +25,11 @@ var Certs = &cli.Command{
Usage: "remove a certificate from the database",
Action: removeCert,
},
{
Name: "migrate",
Usage: "migrate from \"pogreb\" driver to dbms driver",
Action: migrateCerts,
},
},
Flags: []cli.Flag{
// Cert Storage
@ -35,35 +42,83 @@ var Certs = &cli.Command{
},
&cli.StringFlag{
Name: "db-type",
Value: "sqlite",
Value: "", // TODO: "sqlite3" in next version
EnvVars: []string{"DB_TYPE"},
},
&cli.StringFlag{
Name: "db-conn",
Value: "", // TODO: "certs.sqlite", in next version
Value: "certs.sqlite",
EnvVars: []string{"DB_CONN"},
},
&cli.BoolFlag{
Name: "verbose",
Usage: "print trace info",
EnvVars: []string{"VERBOSE"},
Value: false,
},
},
}
func listCerts(ctx *cli.Context) error {
keyDatabase, err := database.NewPogreb(ctx.String("db-pogreb"))
if err != nil {
return fmt.Errorf("could not create database: %v", err)
func migrateCerts(ctx *cli.Context) error {
dbType := ctx.String("db-type")
if dbType == "" {
dbType = "sqlite3"
}
dbConn := ctx.String("db-conn")
dbPogrebConn := ctx.String("db-pogreb")
verbose := ctx.Bool("verbose")
log.Level(zerolog.InfoLevel)
if verbose {
log.Level(zerolog.TraceLevel)
}
items, err := keyDatabase.Items(0, 0)
xormDB, err := database.NewXormDB(dbType, dbConn)
if err != nil {
return fmt.Errorf("could not connect to database: %w", err)
}
defer xormDB.Close()
pogrebDB, err := database.NewPogreb(dbPogrebConn)
if err != nil {
return fmt.Errorf("could not open database: %w", err)
}
defer pogrebDB.Close()
fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
certs, err := pogrebDB.Items(0, 0)
if err != nil {
return err
}
fmt.Printf("Name\tDomain\tValidTill\n\n")
for _, cert := range items {
if cert.Name[0] == '.' {
cert.Name = "*" + cert.Name
for _, cert := range certs {
if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
return err
}
fmt.Printf("%s\t%s\t%s\n",
cert.Name,
}
fmt.Println("... done")
return nil
}
func listCerts(ctx *cli.Context) error {
certDB, err := openCertDB(ctx)
if err != nil {
return err
}
items, err := certDB.Items(0, 0)
if err != nil {
return err
}
fmt.Printf("Domain\tValidTill\n\n")
for _, cert := range items {
if cert.Domain[0] == '.' {
cert.Domain = "*" + cert.Domain
}
fmt.Printf("%s\t%s\n",
cert.Domain,
time.Unix(cert.ValidTill, 0).Format(time.RFC3339))
}
@ -77,19 +132,48 @@ func removeCert(ctx *cli.Context) error {
domains := ctx.Args().Slice()
keyDatabase, err := database.NewPogreb(ctx.String("db-pogreb"))
certDB, err := openCertDB(ctx)
if err != nil {
return fmt.Errorf("could not create database: %v", err)
return err
}
for _, domain := range domains {
fmt.Printf("Removing domain %s from the database...\n", domain)
if err := keyDatabase.Delete(domain); err != nil {
if err := certDB.Delete(domain); err != nil {
return err
}
}
if err := keyDatabase.Close(); err != nil {
if err := certDB.Close(); err != nil {
return err
}
return nil
}
func openCertDB(ctx *cli.Context) (certDB database.CertDB, err error) {
if ctx.String("db-type") != "" {
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, fmt.Errorf("could not connect to database: %w", err)
}
} else {
// TODO: remove in next version
fmt.Println(`
######################
## W A R N I N G !!! #
######################
You use "pogreb" witch is deprecated and will be removed in the next version.
Please switch to sqlite, mysql or postgres !!!
The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.
`)
log.Error().Msg("depricated \"pogreb\" used\n")
certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
if err != nil {
return nil, fmt.Errorf("could not create database: %w", err)
}
}
return certDB, nil
}

4
cmd/flags.go
View File

@ -131,12 +131,12 @@ var ServeFlags = []cli.Flag{
},
&cli.StringFlag{
Name: "db-type",
Value: "sqlite",
Value: "", // TODO: "sqlite3" in next version
EnvVars: []string{"DB_TYPE"},
},
&cli.StringFlag{
Name: "db-conn",
Value: "", // TODO: "certs.sqlite", in next version
Value: "certs.sqlite",
EnvVars: []string{"DB_CONN"},
},
}

32
cmd/main.go
View File

@ -75,12 +75,34 @@ func Serve(ctx *cli.Context) error {
}
// Init ssl cert database
dbPogrebDB := ctx.String("db-pogreb")
certDB, err := database.NewPogreb(dbPogrebDB)
if err != nil {
return fmt.Errorf("could not create database: %v", err)
var certDB database.CertDB
if ctx.String("db-type") != "" {
log.Trace().Msg("use xorm mode")
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return fmt.Errorf("could not connect to database: %w", err)
}
} else {
// TODO: remove in next version
fmt.Println(`
######################
## W A R N I N G !!! #
######################
You use "pogreb" witch is deprecated and will be removed in the next version.
Please switch to sqlite, mysql or postgres !!!
The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.
`)
log.Error().Msg("depricated \"pogreb\" used\n")
certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
if err != nil {
return fmt.Errorf("could not create database: %w", err)
}
}
defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
defer certDB.Close()
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()

6
server/certificates/certificates.go
View File

@ -485,11 +485,11 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
log.Error().Err(err).Msg("could not get certs from list")
} else {
for _, cert := range certs {
if !strings.EqualFold(cert.Name, mainDomainSuffix) {
if !strings.EqualFold(cert.Domain, strings.TrimPrefix(mainDomainSuffix, ".")) {
if time.Unix(cert.ValidTill, 0).Before(threshold) {
err := certDB.Delete(cert.Name)
err := certDB.Delete(cert.Domain)
if err != nil {
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Name)
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Domain)
} else {
expiredCertCount++
}

52
server/database/interface.go
View File

@ -19,29 +19,28 @@ type CertDB interface {
}
type Cert struct {
Name string `xorm:"pk NOT NULL 'name'"`
Domain string `xorm:" NOT NULL UNIQUE 'domain'"` // TODO: check: is name always same as domain?
Domain string `xorm:"pk NOT NULL UNIQUE 'domain'"`
Created int64 `xorm:"created NOT NULL DEFAULT 0 'created'"`
Updated int64 `xorm:"updated NOT NULL DEFAULT 0 'updated'"`
ValidTill int64 `xorm:" NOT NULL DEFAULT 0 'valid_till'"`
// certificate.Resource
certURL string `xorm:"'cert_url'"`
certStableURL string `xorm:"'cert_stable_url''"`
privateKey []byte `xorm:"'private_key'"`
certificate []byte `xorm:"'certificate'"`
issuerCertificate []byte `xorm:"'issuer_certificate'"` // TODO: dedup ?
csr []byte `xorm:"'csr'"`
CertURL string `xorm:"'cert_url'"`
CertStableURL string `xorm:"'cert_stable_url'"`
PrivateKey []byte `xorm:"'private_key'"`
Certificate []byte `xorm:"'certificate'"`
IssuerCertificate []byte `xorm:"'issuer_certificate'"`
CSR []byte `xorm:"'csr'"`
}
func (c Cert) Raw() *certificate.Resource {
return &certificate.Resource{
Domain: c.Domain,
CertURL: c.certURL,
CertStableURL: c.certStableURL,
PrivateKey: c.privateKey,
Certificate: c.certificate,
IssuerCertificate: c.issuerCertificate,
CSR: c.csr,
CertURL: c.CertURL,
CertStableURL: c.CertStableURL,
PrivateKey: c.PrivateKey,
Certificate: c.Certificate,
IssuerCertificate: c.IssuerCertificate,
CSR: c.CSR,
}
}
@ -50,23 +49,28 @@ func toCert(name string, c *certificate.Resource) (*Cert, error) {
if err != nil {
return nil, err
}
if len(tlsCertificates) != 1 || tlsCertificates[0] == nil {
err := fmt.Errorf("parsed cert resource has no or more than one cert")
log.Error().Err(err).Str("name", name).Msgf("cert: %v", c)
if len(tlsCertificates) == 0 || tlsCertificates[0] == nil {
err := fmt.Errorf("parsed cert resource has no cert")
log.Error().Err(err).Str("domain", c.Domain).Msgf("cert: %v", c)
return nil, err
}
validTill := tlsCertificates[0].NotAfter.Unix()
// TODO: do we need this or can we just go with domain name for wildcard cert
// default *.mock cert is prefixed with '.'
if name != c.Domain && name[1:] != c.Domain && name[0] != '.' {
return nil, fmt.Errorf("domain key and cert domain not equal")
}
return &Cert{
Name: name,
Domain: c.Domain,
ValidTill: validTill,
certURL: c.CertURL,
certStableURL: c.CertStableURL,
privateKey: c.PrivateKey,
certificate: c.Certificate,
issuerCertificate: c.IssuerCertificate,
csr: c.CSR,
CertURL: c.CertURL,
CertStableURL: c.CertStableURL,
PrivateKey: c.PrivateKey,
Certificate: c.Certificate,
IssuerCertificate: c.IssuerCertificate,
CSR: c.CSR,
}, nil
}

31
server/database/xorm.go
View File

@ -3,6 +3,7 @@ package database
import (
"errors"
"fmt"
"strings"
"github.com/rs/zerolog/log"
@ -27,6 +28,9 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) {
if !supportedDriver(dbType) {
return nil, fmt.Errorf("not supported db type '%s'", dbType)
}
if dbConn == "" {
return nil, fmt.Errorf("no db connection provided")
}
e, err := xorm.NewEngine(dbType, dbConn)
if err != nil {
@ -46,31 +50,35 @@ func (x xDB) Close() error {
return x.engine.Close()
}
func (x xDB) Put(name string, cert *certificate.Resource) error {
log.Trace().Str("name", name).Msg("inserting cert to db")
c, err := toCert(name, cert)
func (x xDB) Put(domain string, cert *certificate.Resource) error {
log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db")
c, err := toCert(domain, cert)
if err != nil {
return err
}
_, err = x.engine.Insert(c)
return err
}
func (x xDB) Get(name string) (*certificate.Resource, error) {
func (x xDB) Get(domain string) (*certificate.Resource, error) {
// TODO: do we need this or can we just go with domain name for wildcard cert
domain = strings.TrimPrefix(domain, ".")
cert := new(Cert)
log.Trace().Str("name", name).Msg("get cert from db")
if _, err := x.engine.ID(name).Get(&cert); err != nil {
log.Trace().Str("domain", domain).Msg("get cert from db")
if _, err := x.engine.ID(domain).Get(&cert); err != nil {
return nil, err
}
if cert == nil {
return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, name)
return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, domain)
}
return cert.Raw(), nil
}
func (x xDB) Delete(name string) error {
log.Trace().Str("name", name).Msg("delete cert from db")
_, err := x.engine.ID(name).Delete(new(Cert))
func (x xDB) Delete(domain string) error {
log.Trace().Str("domain", domain).Msg("delete cert from db")
_, err := x.engine.ID(domain).Delete(new(Cert))
return err
}
@ -93,7 +101,8 @@ func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
// return all
certs := make([]*Cert, 0, 64)
return certs, x.engine.Find(&certs)
err := x.engine.Find(&certs)
return certs, err
}
// Supported database drivers