diff --git a/cmd/flags.go b/cmd/flags.go index 3c1ac99..8ac09ec 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -63,6 +63,19 @@ var ServeFlags = []cli.Flag{ // TODO: desc EnvVars: []string{"ENABLE_HTTP_SERVER"}, }, + // Server Options + &cli.BoolFlag{ + Name: "enable-lfs-support", + Usage: "enable lfs support, require gitea v1.17.0 as backend", + EnvVars: []string{"ENABLE_LFS_SUPPORT"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "enable-symlink-support", + Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend", + EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, + Value: true, + }, &cli.StringFlag{ Name: "log-level", Value: "warn", diff --git a/cmd/main.go b/cmd/main.go index f57eb60..41809cb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -85,7 +85,7 @@ func Serve(ctx *cli.Context) error { // TODO: make this an MRU cache with a size limit fileResponseCache := cache.NewKeyValueCache() - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken) + giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) if err != nil { return fmt.Errorf("could not create new gitea client: %v", err) } diff --git a/go.mod b/go.mod index 64288de..479c328 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-acme/lego/v4 v4.5.3 github.com/joho/godotenv v1.4.0 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad + github.com/rs/zerolog v1.27.0 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 github.com/valyala/fasthttp v1.31.0 @@ -92,7 +93,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/otp v1.3.0 // indirect - github.com/rs/zerolog v1.27.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect github.com/sacloud/libsacloud v1.36.2 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect diff --git a/go.sum b/go.sum index 4ac6eb6..23a58bc 100644 --- a/go.sum +++ b/go.sum @@ -327,7 +327,6 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -671,8 +670,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/integration/get_test.go b/integration/get_test.go index 191fa3f..6054e17 100644 --- a/integration/get_test.go +++ b/integration/get_test.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/http/cookiejar" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -88,6 +89,34 @@ func TestGetNotFound(t *testing.T) { assert.EqualValues(t, 37, getSize(resp.Body)) } +func TestFollowSymlink(t *testing.T) { + log.Printf("=== TestFollowSymlink ===\n") + + resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link") + assert.NoError(t, err) + if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + t.FailNow() + } + assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type")) + assert.EqualValues(t, "4", resp.Header.Get("Content-Length")) + body := getBytes(resp.Body) + assert.EqualValues(t, 4, len(body)) + assert.EqualValues(t, "abc\n", string(body)) +} + +func TestLFSSupport(t *testing.T) { + log.Printf("=== TestLFSSupport ===\n") + + resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt") + assert.NoError(t, err) + if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) { + t.FailNow() + } + body := strings.TrimSpace(string(getBytes(resp.Body))) + assert.EqualValues(t, 12, len(body)) + assert.EqualValues(t, "actual value", body) +} + func getTestHTTPSClient() *http.Client { cookieJar, _ := cookiejar.New(nil) return &http.Client{ @@ -101,6 +130,12 @@ func getTestHTTPSClient() *http.Client { } } +func getBytes(stream io.Reader) []byte { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(stream) + return buf.Bytes() +} + func getSize(stream io.Reader) int { buf := new(bytes.Buffer) _, _ = buf.ReadFrom(stream) diff --git a/server/gitea/cache.go b/server/gitea/cache.go new file mode 100644 index 0000000..932ff3c --- /dev/null +++ b/server/gitea/cache.go @@ -0,0 +1,12 @@ +package gitea + +type FileResponse struct { + Exists bool + ETag []byte + MimeType string + Body []byte +} + +func (f FileResponse) IsEmpty() bool { + return len(f.Body) != 0 +} diff --git a/server/gitea/client.go b/server/gitea/client.go index 3b9ad6f..16cba84 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -7,11 +7,15 @@ import ( "strings" "time" + "github.com/rs/zerolog/log" "github.com/valyala/fasthttp" "github.com/valyala/fastjson" ) -const giteaAPIRepos = "/api/v1/repos/" +const ( + giteaAPIRepos = "/api/v1/repos/" + giteaObjectTypeHeader = "X-Gitea-Object-Type" +) var ErrorNotFound = errors.New("not found") @@ -21,13 +25,9 @@ type Client struct { fastClient *fasthttp.Client infoTimeout time.Duration contentTimeout time.Duration -} -type FileResponse struct { - Exists bool - ETag []byte - MimeType string - Body []byte + followSymlinks bool + supportLFS bool } // TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package @@ -44,9 +44,7 @@ func joinURL(baseURL string, paths ...string) string { return baseURL + "/" + strings.Join(p, "/") } -func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 } - -func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { +func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) { rootURL, err := url.Parse(giteaRoot) giteaRoot = strings.Trim(rootURL.String(), "/") @@ -56,29 +54,28 @@ func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) { infoTimeout: 5 * time.Second, contentTimeout: 10 * time.Second, fastClient: getFastHTTPClient(), + + followSymlinks: followSymlinks, + supportLFS: supportLFS, }, err } func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) - res, err := client.do(client.contentTimeout, url) + resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource) if err != nil { return nil, err } - - switch res.StatusCode() { - case fasthttp.StatusOK: - return res.Body(), nil - case fasthttp.StatusNotFound: - return nil, ErrorNotFound - default: - return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) - } + return resp.Body(), nil } -func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) { - url := joinURL(client.giteaRoot, giteaAPIRepos, uri) - res, err := client.do(client.contentTimeout, url) +func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) { + var apiURL string + if client.supportLFS { + apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref)) + } else { + apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)) + } + resp, err := client.do(client.contentTimeout, apiURL) if err != nil { return nil, err } @@ -87,13 +84,24 @@ func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) { return nil, err } - switch res.StatusCode() { + switch resp.StatusCode() { case fasthttp.StatusOK: - return res, nil + objType := string(resp.Header.Peek(giteaObjectTypeHeader)) + log.Trace().Msgf("server raw content object: %s", objType) + if client.followSymlinks && objType == "symlink" { + // TODO: limit to 1000 chars if we switched to std + linkDest := strings.TrimSpace(string(resp.Body())) + log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest) + return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest) + } + + return resp, nil + case fasthttp.StatusNotFound: return nil, ErrorNotFound + default: - return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode()) + return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode()) } } diff --git a/server/handler_test.go b/server/handler_test.go index 23d9af5..f9a721a 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -13,7 +13,7 @@ import ( func TestHandlerPerformance(t *testing.T) { giteaRoot := "https://codeberg.org" - giteaClient, _ := gitea.NewClient(giteaRoot, "") + giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false) testHandler := Handler( []byte("codeberg.page"), []byte("raw.codeberg.org"), giteaClient, diff --git a/server/upstream/helper.go b/server/upstream/helper.go index 5bbe833..0714dcd 100644 --- a/server/upstream/helper.go +++ b/server/upstream/helper.go @@ -67,6 +67,10 @@ func (o *Options) generateUri() string { return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath) } +func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) { + return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath +} + func (o *Options) timestamp() string { return strconv.FormatInt(o.BranchTimestamp.Unix(), 10) } diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 4371e88..0e27727 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -83,7 +83,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() { cachedResponse = cachedValue.(gitea.FileResponse) } else { - res, err = giteaClient.ServeRawContent(uri) + res, err = giteaClient.ServeRawContent(o.generateUriClientArgs()) } log.Debug().Msg("Aquisting")