diff --git a/.gitignore b/.gitignore index dfe69ac..60c6d76 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ acme-account.json build/ vendor/ pages +certs.sqlite diff --git a/Dockerfile b/Dockerfile index 380c373..eec97de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,6 @@ FROM techknowlogick/xgo as build WORKDIR /workspace -RUN apk add ca-certificates COPY . . RUN CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' . diff --git a/Justfile b/Justfile index 579a604..2f40499 100644 --- a/Justfile +++ b/Justfile @@ -1,3 +1,6 @@ +CGO_FLAGS := '-extldflags "-static" -linkmode external' +TAGS := 'sqlite sqlite_unlock_notify netgo' + dev: #!/usr/bin/env bash set -euxo pipefail @@ -7,13 +10,13 @@ dev: export RAW_DOMAIN=raw.localhost.mock.directory export PORT=4430 export LOG_LEVEL=trace - go run -tags 'sqlite sqlite_unlock_notify -ldflags '-s -w -extldflags "-static" -linkmode external' netgo' . + go run -tags 'sqlite sqlite_unlock_notify netgo' . build: - CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' -ldflags '-s -w -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags '{{TAGS}}' -ldflags '-s -w {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./ build-tag VERSION: - CGO_ENABLED=1 go build -tags 'sqlite sqlite_unlock_notify netgo' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" -extldflags "-static" -linkmode external' -v -o build/codeberg-pages-server ./ + CGO_ENABLED=1 go build -tags '{{TAGS}}' '-ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}" {{CGO_FLAGS}}' -v -o build/codeberg-pages-server ./ lint: tool-golangci tool-gofumpt [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \ @@ -38,13 +41,13 @@ tool-gofumpt: fi test: - go test -race -tags 'sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ test-run TEST: - go test -race -tags 'sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ integration: - go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/... integration-run TEST: - go test -race -tags 'integration sqlite sqlite_unlock_notify netgo' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... + go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/... diff --git a/cmd/certs.go b/cmd/certs.go index 46a9c96..3bbab36 100644 --- a/cmd/certs.go +++ b/cmd/certs.go @@ -4,6 +4,8 @@ import ( "fmt" "time" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "codeberg.org/codeberg/pages/server/database" @@ -23,6 +25,11 @@ var Certs = &cli.Command{ Usage: "remove a certificate from the database", Action: removeCert, }, + { + Name: "migrate", + Usage: "migrate from \"pogreb\" driver to dbms driver", + Action: migrateCerts, + }, }, Flags: []cli.Flag{ // Cert Storage @@ -35,35 +42,83 @@ var Certs = &cli.Command{ }, &cli.StringFlag{ Name: "db-type", - Value: "sqlite", + Value: "", // TODO: "sqlite3" in next version EnvVars: []string{"DB_TYPE"}, }, &cli.StringFlag{ Name: "db-conn", - Value: "", // TODO: "certs.sqlite", in next version + Value: "certs.sqlite", EnvVars: []string{"DB_CONN"}, }, + &cli.BoolFlag{ + Name: "verbose", + Usage: "print trace info", + EnvVars: []string{"VERBOSE"}, + Value: false, + }, }, } -func listCerts(ctx *cli.Context) error { - keyDatabase, err := database.NewPogreb(ctx.String("db-pogreb")) - if err != nil { - return fmt.Errorf("could not create database: %v", err) +func migrateCerts(ctx *cli.Context) error { + dbType := ctx.String("db-type") + if dbType == "" { + dbType = "sqlite3" + } + dbConn := ctx.String("db-conn") + dbPogrebConn := ctx.String("db-pogreb") + verbose := ctx.Bool("verbose") + + log.Level(zerolog.InfoLevel) + if verbose { + log.Level(zerolog.TraceLevel) } - items, err := keyDatabase.Items(0, 0) + xormDB, err := database.NewXormDB(dbType, dbConn) + if err != nil { + return fmt.Errorf("could not connect to database: %w", err) + } + defer xormDB.Close() + + pogrebDB, err := database.NewPogreb(dbPogrebConn) + if err != nil { + return fmt.Errorf("could not open database: %w", err) + } + defer pogrebDB.Close() + + fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn) + + certs, err := pogrebDB.Items(0, 0) if err != nil { return err } - fmt.Printf("Name\tDomain\tValidTill\n\n") - for _, cert := range items { - if cert.Name[0] == '.' { - cert.Name = "*" + cert.Name + for _, cert := range certs { + if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil { + return err } - fmt.Printf("%s\t%s\t%s\n", - cert.Name, + } + + fmt.Println("... done") + return nil +} + +func listCerts(ctx *cli.Context) error { + certDB, err := openCertDB(ctx) + if err != nil { + return err + } + + items, err := certDB.Items(0, 0) + if err != nil { + return err + } + + fmt.Printf("Domain\tValidTill\n\n") + for _, cert := range items { + if cert.Domain[0] == '.' { + cert.Domain = "*" + cert.Domain + } + fmt.Printf("%s\t%s\n", cert.Domain, time.Unix(cert.ValidTill, 0).Format(time.RFC3339)) } @@ -77,19 +132,48 @@ func removeCert(ctx *cli.Context) error { domains := ctx.Args().Slice() - keyDatabase, err := database.NewPogreb(ctx.String("db-pogreb")) + certDB, err := openCertDB(ctx) if err != nil { - return fmt.Errorf("could not create database: %v", err) + return err } for _, domain := range domains { fmt.Printf("Removing domain %s from the database...\n", domain) - if err := keyDatabase.Delete(domain); err != nil { + if err := certDB.Delete(domain); err != nil { return err } } - if err := keyDatabase.Close(); err != nil { + if err := certDB.Close(); err != nil { return err } return nil } + +func openCertDB(ctx *cli.Context) (certDB database.CertDB, err error) { + if ctx.String("db-type") != "" { + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, fmt.Errorf("could not connect to database: %w", err) + } + } else { + // TODO: remove in next version + fmt.Println(` +###################### +## W A R N I N G !!! # +###################### + +You use "pogreb" witch is deprecated and will be removed in the next version. +Please switch to sqlite, mysql or postgres !!! + +The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start. + +`) + log.Error().Msg("depricated \"pogreb\" used\n") + + certDB, err = database.NewPogreb(ctx.String("db-pogreb")) + if err != nil { + return nil, fmt.Errorf("could not create database: %w", err) + } + } + return certDB, nil +} diff --git a/cmd/flags.go b/cmd/flags.go index 87e28ea..e2b4bbf 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -131,12 +131,12 @@ var ServeFlags = []cli.Flag{ }, &cli.StringFlag{ Name: "db-type", - Value: "sqlite", + Value: "", // TODO: "sqlite3" in next version EnvVars: []string{"DB_TYPE"}, }, &cli.StringFlag{ Name: "db-conn", - Value: "", // TODO: "certs.sqlite", in next version + Value: "certs.sqlite", EnvVars: []string{"DB_CONN"}, }, } diff --git a/cmd/main.go b/cmd/main.go index fc7afd4..263c3df 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -75,12 +75,34 @@ func Serve(ctx *cli.Context) error { } // Init ssl cert database - dbPogrebDB := ctx.String("db-pogreb") - certDB, err := database.NewPogreb(dbPogrebDB) - if err != nil { - return fmt.Errorf("could not create database: %v", err) + var certDB database.CertDB + if ctx.String("db-type") != "" { + log.Trace().Msg("use xorm mode") + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return fmt.Errorf("could not connect to database: %w", err) + } + } else { + // TODO: remove in next version + fmt.Println(` +###################### +## W A R N I N G !!! # +###################### + +You use "pogreb" witch is deprecated and will be removed in the next version. +Please switch to sqlite, mysql or postgres !!! + +The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start. + +`) + log.Error().Msg("depricated \"pogreb\" used\n") + + certDB, err = database.NewPogreb(ctx.String("db-pogreb")) + if err != nil { + return fmt.Errorf("could not create database: %w", err) + } } - defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it + defer certDB.Close() keyCache := cache.NewKeyValueCache() challengeCache := cache.NewKeyValueCache() diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 3096c3c..62b2e64 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -485,11 +485,11 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi log.Error().Err(err).Msg("could not get certs from list") } else { for _, cert := range certs { - if !strings.EqualFold(cert.Name, mainDomainSuffix) { + if !strings.EqualFold(cert.Domain, strings.TrimPrefix(mainDomainSuffix, ".")) { if time.Unix(cert.ValidTill, 0).Before(threshold) { - err := certDB.Delete(cert.Name) + err := certDB.Delete(cert.Domain) if err != nil { - log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Name) + log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", cert.Domain) } else { expiredCertCount++ } diff --git a/server/database/interface.go b/server/database/interface.go index 15d5de6..b551c75 100644 --- a/server/database/interface.go +++ b/server/database/interface.go @@ -19,29 +19,28 @@ type CertDB interface { } type Cert struct { - Name string `xorm:"pk NOT NULL 'name'"` - Domain string `xorm:" NOT NULL UNIQUE 'domain'"` // TODO: check: is name always same as domain? + Domain string `xorm:"pk NOT NULL UNIQUE 'domain'"` Created int64 `xorm:"created NOT NULL DEFAULT 0 'created'"` Updated int64 `xorm:"updated NOT NULL DEFAULT 0 'updated'"` ValidTill int64 `xorm:" NOT NULL DEFAULT 0 'valid_till'"` // certificate.Resource - certURL string `xorm:"'cert_url'"` - certStableURL string `xorm:"'cert_stable_url''"` - privateKey []byte `xorm:"'private_key'"` - certificate []byte `xorm:"'certificate'"` - issuerCertificate []byte `xorm:"'issuer_certificate'"` // TODO: dedup ? - csr []byte `xorm:"'csr'"` + CertURL string `xorm:"'cert_url'"` + CertStableURL string `xorm:"'cert_stable_url'"` + PrivateKey []byte `xorm:"'private_key'"` + Certificate []byte `xorm:"'certificate'"` + IssuerCertificate []byte `xorm:"'issuer_certificate'"` + CSR []byte `xorm:"'csr'"` } func (c Cert) Raw() *certificate.Resource { return &certificate.Resource{ Domain: c.Domain, - CertURL: c.certURL, - CertStableURL: c.certStableURL, - PrivateKey: c.privateKey, - Certificate: c.certificate, - IssuerCertificate: c.issuerCertificate, - CSR: c.csr, + CertURL: c.CertURL, + CertStableURL: c.CertStableURL, + PrivateKey: c.PrivateKey, + Certificate: c.Certificate, + IssuerCertificate: c.IssuerCertificate, + CSR: c.CSR, } } @@ -50,23 +49,28 @@ func toCert(name string, c *certificate.Resource) (*Cert, error) { if err != nil { return nil, err } - if len(tlsCertificates) != 1 || tlsCertificates[0] == nil { - err := fmt.Errorf("parsed cert resource has no or more than one cert") - log.Error().Err(err).Str("name", name).Msgf("cert: %v", c) + if len(tlsCertificates) == 0 || tlsCertificates[0] == nil { + err := fmt.Errorf("parsed cert resource has no cert") + log.Error().Err(err).Str("domain", c.Domain).Msgf("cert: %v", c) return nil, err } validTill := tlsCertificates[0].NotAfter.Unix() + // TODO: do we need this or can we just go with domain name for wildcard cert + // default *.mock cert is prefixed with '.' + if name != c.Domain && name[1:] != c.Domain && name[0] != '.' { + return nil, fmt.Errorf("domain key and cert domain not equal") + } + return &Cert{ - Name: name, Domain: c.Domain, ValidTill: validTill, - certURL: c.CertURL, - certStableURL: c.CertStableURL, - privateKey: c.PrivateKey, - certificate: c.Certificate, - issuerCertificate: c.IssuerCertificate, - csr: c.CSR, + CertURL: c.CertURL, + CertStableURL: c.CertStableURL, + PrivateKey: c.PrivateKey, + Certificate: c.Certificate, + IssuerCertificate: c.IssuerCertificate, + CSR: c.CSR, }, nil } diff --git a/server/database/xorm.go b/server/database/xorm.go index 9e4ed84..5e7296c 100644 --- a/server/database/xorm.go +++ b/server/database/xorm.go @@ -3,6 +3,7 @@ package database import ( "errors" "fmt" + "strings" "github.com/rs/zerolog/log" @@ -27,6 +28,9 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) { if !supportedDriver(dbType) { return nil, fmt.Errorf("not supported db type '%s'", dbType) } + if dbConn == "" { + return nil, fmt.Errorf("no db connection provided") + } e, err := xorm.NewEngine(dbType, dbConn) if err != nil { @@ -46,31 +50,35 @@ func (x xDB) Close() error { return x.engine.Close() } -func (x xDB) Put(name string, cert *certificate.Resource) error { - log.Trace().Str("name", name).Msg("inserting cert to db") - c, err := toCert(name, cert) +func (x xDB) Put(domain string, cert *certificate.Resource) error { + log.Trace().Str("domain", cert.Domain).Msg("inserting cert to db") + c, err := toCert(domain, cert) if err != nil { return err } + _, err = x.engine.Insert(c) return err } -func (x xDB) Get(name string) (*certificate.Resource, error) { +func (x xDB) Get(domain string) (*certificate.Resource, error) { + // TODO: do we need this or can we just go with domain name for wildcard cert + domain = strings.TrimPrefix(domain, ".") + cert := new(Cert) - log.Trace().Str("name", name).Msg("get cert from db") - if _, err := x.engine.ID(name).Get(&cert); err != nil { + log.Trace().Str("domain", domain).Msg("get cert from db") + if _, err := x.engine.ID(domain).Get(&cert); err != nil { return nil, err } if cert == nil { - return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, name) + return nil, fmt.Errorf("%w: name='%s'", ErrNotFound, domain) } return cert.Raw(), nil } -func (x xDB) Delete(name string) error { - log.Trace().Str("name", name).Msg("delete cert from db") - _, err := x.engine.ID(name).Delete(new(Cert)) +func (x xDB) Delete(domain string) error { + log.Trace().Str("domain", domain).Msg("delete cert from db") + _, err := x.engine.ID(domain).Delete(new(Cert)) return err } @@ -93,7 +101,8 @@ func (x xDB) Items(page, pageSize int) ([]*Cert, error) { // return all certs := make([]*Cert, 0, 64) - return certs, x.engine.Find(&certs) + err := x.engine.Find(&certs) + return certs, err } // Supported database drivers