Merge branch 'std-http' into refactor_split-long-funcs

.woodpecker.yml

@@ -15,6 +15,10 @@ pipeline:
       - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
       - golangci-lint run --timeout 5m --build-tags integration
 
+  editor-config:
+    group: compliant
+    image: mstruebing/editorconfig-checker
+
   build:
     group: compliant
     image: codeberg.org/6543/docker-images/golang_just
@@ -90,6 +94,7 @@ pipeline:
         from_secret: bot_token
     when:
       event: [ "push" ]
+      branch: ${CI_REPO_DEFAULT_BRANCH}
 
   docker-tag:
     image: plugins/kaniko

html/error.go

@@ -1,7 +1,6 @@
 package html
 
 import (
-	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -9,8 +8,8 @@ import (
 	"codeberg.org/codeberg/pages/server/context"
 )
 
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
-// with the provided status code.
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
+// with "%status%" and %message% replaced with the provided statusCode and msg
 func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
 	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
 	ctx.RespWriter.WriteHeader(statusCode)
@@ -22,7 +21,7 @@ func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
 		msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode))
 	}
 
-	_, _ = io.Copy(ctx.RespWriter, strings.NewReader(msg))
+	_, _ = ctx.RespWriter.Write([]byte(msg))
 }
 
 func errorMessage(statusCode int) string {

server/context/context.go

@@ -2,9 +2,7 @@ package context
 
 import (
 	stdContext "context"
-	"io"
 	"net/http"
-	"strings"
 )
 
 type Context struct {
@@ -41,7 +39,7 @@ func (c *Context) String(raw string, status ...int) {
 		code = status[0]
 	}
 	c.RespWriter.WriteHeader(code)
-	_, _ = io.Copy(c.RespWriter, strings.NewReader(raw))
+	_, _ = c.RespWriter.Write([]byte(raw))
 }
 
 func (c *Context) IsMethod(m string) bool {

server/database/mock.go

@@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error {
 func (p tmpDB) Get(name string) (*certificate.Resource, error) {
 	cert, has := p.intern.Get(name)
 	if !has {
-		return nil, fmt.Errorf("cert for '%s' not found", name)
+		return nil, fmt.Errorf("cert for %q not found", name)
 	}
 	return cert.(*certificate.Resource), nil
 }

server/gitea/cache.go

@@ -27,7 +27,7 @@ const (
 	fileCacheTimeout = 5 * time.Minute
 
 	// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-	fileCacheSizeLimit = int64(1024 * 1024)
+	fileCacheSizeLimit = int64(1000 * 1000)
 )
 
 type FileResponse struct {
@@ -60,7 +60,7 @@ func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) {
 	header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
 	header.Set(PagesCacheIndicatorHeader, "true")
 
-	log.Trace().Msgf("fileCache for '%s' used", cacheKey)
+	log.Trace().Msgf("fileCache for %q used", cacheKey)
 	return header, statusCode
 }
 
@@ -71,45 +71,46 @@ type BranchTimestamp struct {
 }
 
 type writeCacheReader struct {
-	rc       io.ReadCloser
-	buff     *bytes.Buffer
-	f        *FileResponse
-	cacheKey string
-	cache    cache.SetGetKey
-	hasErr   bool
+	originalReader io.ReadCloser
+	buffer         *bytes.Buffer
+	rileResponse   *FileResponse
+	cacheKey       string
+	cache          cache.SetGetKey
+	hasError       bool
 }
 
 func (t *writeCacheReader) Read(p []byte) (n int, err error) {
-	n, err = t.rc.Read(p)
+	n, err = t.originalReader.Read(p)
 	if err != nil {
-		t.hasErr = true
+		log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
+		t.hasError = true
 	} else if n > 0 {
-		_, _ = t.buff.Write(p[:n])
+		_, _ = t.buffer.Write(p[:n])
 	}
 	return
 }
 
 func (t *writeCacheReader) Close() error {
-	if !t.hasErr {
-		fc := *t.f
-		fc.Body = t.buff.Bytes()
+	if !t.hasError {
+		fc := *t.rileResponse
+		fc.Body = t.buffer.Bytes()
 		_ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
 	}
-	log.Trace().Msgf("cacheReader for '%s' saved=%v closed", t.cacheKey, !t.hasErr)
-	return t.rc.Close()
+	log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
+	return t.originalReader.Close()
 }
 
 func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
-	buf := []byte{}
 	if r == nil || cache == nil || cacheKey == "" {
 		log.Error().Msg("could not create CacheReader")
-		return r
+		return nil
 	}
+
 	return &writeCacheReader{
-		rc:       r,
-		buff:     bytes.NewBuffer(buf),
-		f:        &f,
-		cache:    cache,
-		cacheKey: cacheKey,
+		originalReader: r,
+		buffer:         bytes.NewBuffer(make([]byte, 0)),
+		rileResponse:   &f,
+		cache:          cache,
+		cacheKey:       cacheKey,
 	}
 }

server/gitea/client.go

@@ -29,6 +29,7 @@ const (
 
 	// pages server
 	PagesCacheIndicatorHeader = "X-Pages-Cache"
+	symlinkReadLimit          = 10000
 
 	// gitea
 	giteaObjectTypeHeader = "X-Gitea-Object-Type"
@@ -107,7 +108,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 		if cache.Exists {
 			if cache.IsSymlink {
 				linkDest := string(cache.Body)
-				log.Debug().Msgf("[cache] follow symlink from'%s' to '%s'", resource, linkDest)
+				log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
 				return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
 			} else {
 				log.Debug().Msg("[cache] return bytes")
@@ -118,8 +119,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 		}
 	}
 
-	// if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
-	// 	cachedResponse = cachedValue.(gitea.FileResponse)
+	// not in cache, open reader via gitea api
 	reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
 	if resp != nil {
 		switch resp.StatusCode {
@@ -127,55 +127,53 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 			// first handle symlinks
 			{
 				objType := resp.Header.Get(giteaObjectTypeHeader)
-				log.Trace().Msgf("server raw content object: %s", objType)
+				log.Trace().Msgf("server raw content object %q", objType)
 				if client.followSymlinks && objType == objTypeSymlink {
-					// limit to 1000 chars
 					defer reader.Close()
-					linkDestBytes, err := io.ReadAll(io.LimitReader(reader, 10000))
+					// read limited chars for symlink
+					linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit))
 					if err != nil {
 						return nil, nil, http.StatusInternalServerError, err
 					}
 					linkDest := strings.TrimSpace(string(linkDestBytes))
 
+					// we store symlink not content to reduce duplicates in cache
 					if err := client.responseCache.Set(cacheKey, FileResponse{
 						Exists:    true,
 						IsSymlink: true,
 						Body:      []byte(linkDest),
 						ETag:      resp.Header.Get(ETagHeader),
 					}, fileCacheTimeout); err != nil {
-						log.Error().Err(err).Msg("could not save symlink in cache")
+						log.Error().Err(err).Msg("[cache] error on cache write")
 					}
 
-					log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+					log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
 					return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
 				}
 			}
 
-			// now we are sure it's content
-			{
-				// Set the MIME type
-				mimeType := client.getMimeTypeByExtension(resource)
-				resp.Response.Header.Set(ContentTypeHeader, mimeType)
+			// now we are sure it's content so set the MIME type
+			mimeType := client.getMimeTypeByExtension(resource)
+			resp.Response.Header.Set(ContentTypeHeader, mimeType)
 
-				if !shouldRespBeSavedToCache(resp.Response) {
-					return reader, resp.Response.Header, resp.StatusCode, err
-				}
-
-				// now we write to cache and respond at the sime time
-				fileResp := FileResponse{
-					Exists:   true,
-					ETag:     resp.Header.Get(ETagHeader),
-					MimeType: mimeType,
-				}
-				return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
+			if !shouldRespBeSavedToCache(resp.Response) {
+				return reader, resp.Response.Header, resp.StatusCode, err
 			}
 
+			// now we write to cache and respond at the sime time
+			fileResp := FileResponse{
+				Exists:   true,
+				ETag:     resp.Header.Get(ETagHeader),
+				MimeType: mimeType,
+			}
+			return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
+
 		case http.StatusNotFound:
 			if err := client.responseCache.Set(cacheKey, FileResponse{
 				Exists: false,
 				ETag:   resp.Header.Get(ETagHeader),
 			}, fileCacheTimeout); err != nil {
-				log.Error().Err(err).Msg("could not save 404 in cache")
+				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 
 			return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound
@@ -192,19 +190,19 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 	if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
 		branchTimeStamp := stamp.(*BranchTimestamp)
 		if branchTimeStamp.notFound {
-			log.Trace().Msgf("use cache branch [%s] not found", branchName)
+			log.Trace().Msgf("[cache] use branch %q not found", branchName)
 			return &BranchTimestamp{}, ErrorNotFound
 		}
-		log.Trace().Msgf("use cache branch [%s] exist", branchName)
+		log.Trace().Msgf("[cache] use branch %q exist", branchName)
 		return branchTimeStamp, nil
 	}
 
 	branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
-			log.Trace().Msgf("set cache branch [%s] not found", branchName)
+			log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
 			if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
-				log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName)
+				log.Error().Err(err).Msg("[cache] error on cache write")
 			}
 			return &BranchTimestamp{}, ErrorNotFound
 		}
@@ -221,7 +219,7 @@ func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchNam
 
 	log.Trace().Msgf("set cache branch [%s] exist", branchName)
 	if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
-		log.Error().Err(err).Msgf("error on store of repo branch timestamp [%s/%s@%s]", repoOwner, repoName, branchName)
+		log.Error().Err(err).Msg("[cache] error on cache write")
 	}
 	return stamp, nil
 }
@@ -243,7 +241,7 @@ func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (str
 
 	branch := repo.DefaultBranch
 	if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
-		log.Error().Err(err).Msgf("error on store of repo default branch [%s/%s]", repoOwner, repoName)
+		log.Error().Err(err).Msg("[cache] error on cache write")
 	}
 	return branch, nil
 }
@@ -254,7 +252,7 @@ func (client *Client) getMimeTypeByExtension(resource string) string {
 	if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
 		mimeType = client.defaultMimeType
 	}
-	log.Trace().Msgf("probe mime: %s", mimeType)
+	log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
 	return mimeType
 }
 
@@ -263,12 +261,12 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
 		return false
 	}
 
-	contentLengRaw := resp.Header.Get(ContentLengthHeader)
-	if contentLengRaw == "" {
+	contentLengthRaw := resp.Header.Get(ContentLengthHeader)
+	if contentLengthRaw == "" {
 		return false
 	}
 
-	contentLeng, err := strconv.ParseInt(contentLengRaw, 10, 64)
+	contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
 	if err != nil {
 		log.Error().Err(err).Msg("could not parse content length")
 	}

server/handler.go

@@ -95,7 +95,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 		// TODO: move into external func to not alert vars indirectly
 		tryBranch1 := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool {
 			if repo == "" {
-				log.Debug().Msg("tryBranch: repo == ''")
+				log.Debug().Msg("tryBranch: repo is empty")
 				return false
 			}
 
@@ -213,7 +213,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 						canonicalDomainCache)
 				} else {
 					html.ReturnErrorPage(ctx,
-						fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, targetRepo),
+						fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo),
 						http.StatusFailedDependency)
 				}
 				return
@@ -232,7 +232,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 						canonicalDomainCache)
 				} else {
 					html.ReturnErrorPage(ctx,
-						fmt.Sprintf("explizite set branch '%s' do not exist at '%s/%s'", branch, targetOwner, "pages"),
+						fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"),
 						http.StatusFailedDependency)
 				}
 				return

server/upstream/helper.go

@@ -20,7 +20,7 @@ func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *
 			log.Err(err).Msg("Could't fetch default branch from repository")
 			return nil
 		}
-		log.Debug().Msgf("Succesfully fetched default branch '%s' from Gitea", defaultBranch)
+		log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
 		branch = defaultBranch
 	}
 

server/upstream/upstream.go

@@ -66,7 +66,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 
 		if branch == nil || branch.Branch == "" {
 			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("could not get timestamp of branch '%s'", o.TargetBranch),
+				fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
 				http.StatusFailedDependency)
 			return true
 		}
@@ -182,7 +182,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 	if reader != nil {
 		_, err := io.Copy(ctx.RespWriter, reader)
 		if err != nil {
-			log.Printf("Couldn't write body: %s\n", err)
+			log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
 			html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
 			return true
 		}