User agent

.github/workflows/reproducibility.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          # goreleaser and the ldflags below read the commit date
+          # goreleaser and the Go version stamp read the git history
           fetch-depth: 0
           persist-credentials: false
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
@@ -30,14 +30,12 @@ jobs:
 
       - name: Rebuild from source and compare digests
         run: |
-          VERSION=$(jq -r .version dist/metadata.json)
-          DATE=$(git log -1 --format=%cd --date=format-local:'%Y-%m-%dT%H:%M:%SZ')
           cp -a . /tmp/src-elsewhere
           cd /tmp/src-elsewhere
           CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOAMD64=v1 \
             go build -trimpath \
-              -ldflags="-s -w -buildid= -X main.version=$VERSION -X main.date=$DATE" \
-              -o /tmp/plain-build ./publiccode-parser/publiccode_parser.go
+              -ldflags="-s -w -buildid=" \
+              -o /tmp/plain-build ./publiccode-parser
           sha256sum /tmp/goreleaser-build /tmp/plain-build
           cmp /tmp/goreleaser-build /tmp/plain-build \
             && echo "Reproducible build confirmed"

.gitignore

@@ -1,4 +1,5 @@
 publiccode-parser/publiccode-parser
+dist/
 .vscode
 .history
 .DS_Store

.golangci.yaml

@@ -67,6 +67,9 @@ linters:
         # No point in wrapping these
         - func encoding/json.Marshal(v any)
         - func encoding/json.UnmarshalJSON(v any)
+        # A RoundTripper must return the transport error as is: http.Client
+        # and its callers type assert on it instead of unwrapping it
+        - func (net/http.RoundTripper).RoundTrip(
 
         # Defaults
         - .Errorf(

.goreleaser.yml

@@ -11,11 +11,11 @@ builds:
 -
   id: publiccode-parser
   binary: publiccode-parser
-  main: ./publiccode-parser/publiccode_parser.go
+  main: ./publiccode-parser
   flags:
     - -trimpath
   ldflags:
-    - -s -w -buildid= -X main.version={{.Version}} -X main.date={{.CommitDate}}
+    - -s -w -buildid=
   env:
     - CGO_ENABLED=0
     - SOURCE_DATE_EPOCH={{.CommitTimestamp}}

internal/safehttp.go

@@ -107,7 +107,7 @@ type safeTransport struct {
 func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	resp, err := t.base.RoundTrip(req)
 	if err != nil {
-		return nil, err //nolint:wrapcheck // http.Client inspects the transport error; keep it intact
+		return nil, err
 	}
 
 	// Reject early when the server advertises an over-limit Content-Length.
@@ -122,11 +122,52 @@ func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
+// userAgentTransport sets a User-Agent on the requests that don't carry one,
+// since WAFs often block Go's default "Go-http-client/<version>".
+type userAgentTransport struct {
+	base      http.RoundTripper
+	userAgent string
+}
+
+func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if _, set := req.Header["User-Agent"]; !set && req.Header != nil {
+		// A RoundTripper must not modify the request it is given, hence the clone.
+		req = req.Clone(req.Context())
+		req.Header.Set("User-Agent", t.userAgent)
+	}
+
+	return t.base.RoundTrip(req)
+}
+
+// Option configures the client built by [SafeHTTPClient].
+type Option func(*options)
+
+type options struct {
+	userAgent string
+}
+
+// WithUserAgent sets the User-Agent sent on every request that doesn't already
+// carry one, in place of [UserAgent]. An empty value keeps that default.
+func WithUserAgent(userAgent string) Option {
+	return func(o *options) {
+		o.userAgent = userAgent
+	}
+}
+
 // SafeHTTPClient builds an *http.Client hardened against SSRF and unbounded
 // downloads. When allowPrivate is true the SSRF address filtering is disabled
 // (used for trusted input and tests that target loopback servers); the response
 // size limit is always enforced.
-func SafeHTTPClient(timeout time.Duration, allowPrivate bool) *http.Client {
+func SafeHTTPClient(timeout time.Duration, allowPrivate bool, opts ...Option) *http.Client {
+	o := options{userAgent: UserAgent()}
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.userAgent == "" {
+		o.userAgent = UserAgent()
+	}
+
 	dialer := &net.Dialer{
 		Timeout:   30 * time.Second,
 		KeepAlive: 30 * time.Second,
@@ -146,7 +187,10 @@ func SafeHTTPClient(timeout time.Duration, allowPrivate bool) *http.Client {
 	}
 
 	return &http.Client{
-		Timeout:   timeout,
-		Transport: &safeTransport{base: transport, max: MaxResponseBytes},
+		Timeout: timeout,
+		Transport: &userAgentTransport{
+			base:      &safeTransport{base: transport, max: MaxResponseBytes},
+			userAgent: o.userAgent,
+		},
 	}
 }

internal/safehttp_test.go

@@ -134,3 +134,110 @@ func TestResponseUnderLimitIsReadFully(t *testing.T) {
 		t.Errorf("body mismatch: got %q, want %q", got, body)
 	}
 }
+
+func TestUserAgentTransportSetsTheDefault(t *testing.T) {
+	const userAgent = "libpubliccode/1.2.3 (+https://example.org)"
+
+	var got string
+	transport := &userAgentTransport{
+		base: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			got = r.Header.Get("User-Agent")
+
+			return newResponse("", 0), nil
+		}),
+		userAgent: userAgent,
+	}
+
+	req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	if _, err := transport.RoundTrip(req); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if got != userAgent {
+		t.Errorf("User-Agent sent: got %q, want %q", got, userAgent)
+	}
+
+	// A RoundTripper must not modify the request it is handed.
+	if _, set := req.Header["User-Agent"]; set {
+		t.Errorf("the original request was modified: %q", req.Header.Get("User-Agent"))
+	}
+}
+
+func TestUserAgentTransportKeepsTheCallerHeader(t *testing.T) {
+	cases := map[string]string{
+		"caller sets its own": "harvester/2.0",
+		"caller clears it":    "",
+	}
+
+	for name, want := range cases {
+		t.Run(name, func(t *testing.T) {
+			var got string
+			var seen bool
+
+			transport := &userAgentTransport{
+				base: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+					got, seen = r.Header.Get("User-Agent"), true
+
+					return newResponse("", 0), nil
+				}),
+				userAgent: "libpubliccode/1.2.3",
+			}
+
+			req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil)
+			req.Header["User-Agent"] = []string{want}
+
+			if _, err := transport.RoundTrip(req); err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			if !seen {
+				t.Fatal("the base transport was not called")
+			}
+			if got != want {
+				t.Errorf("User-Agent sent: got %q, want %q", got, want)
+			}
+		})
+	}
+}
+
+// TestSafeHTTPClientSendsUserAgent checks the User-Agent all the way down to the
+// wire, and that the library's own is sent when none is given.
+func TestSafeHTTPClientSendsUserAgent(t *testing.T) {
+	const userAgent = "libpubliccode/1.2.3 (+https://example.org)"
+
+	// The server echoes back what it received, so there is nothing shared
+	// between the handler and the test.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.WriteString(w, r.UserAgent())
+	}))
+	defer srv.Close()
+
+	got, err := getBody(SafeHTTPClient(5*time.Second, true, WithUserAgent(userAgent)), srv.URL)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != userAgent {
+		t.Errorf("User-Agent received by the server: got %q, want %q", got, userAgent)
+	}
+
+	if got, err = getBody(SafeHTTPClient(5*time.Second, true), srv.URL); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != UserAgent() {
+		t.Errorf("User-Agent received by the server: got %q, want the default %q", got, UserAgent())
+	}
+}
+
+// getBody GETs url and returns the response body as a string.
+func getBody(client *http.Client, url string) (string, error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+
+	return string(body), err
+}

internal/useragent.go

@@ -0,0 +1,97 @@
+package netutil
+
+import (
+	"fmt"
+	"runtime/debug"
+	"strings"
+)
+
+const (
+	// productName is the token identifying this software in the User-Agent.
+	productName = "libpubliccode"
+
+	// projectURL is part of the User-Agent so that whoever finds these requests
+	// in their logs can tell what made them, and allow them explicitly instead
+	// of having to allow every Go program.
+	projectURL = "https://github.com/publiccodeyml/libpubliccode"
+
+	// modulePath is this module's import path. The version is looked up under it
+	// in the build information: the parser is the main module when its own CLI
+	// runs, and a dependency when the library is embedded in another program.
+	modulePath = "github.com/publiccodeyml/libpubliccode/v5"
+
+	// unknownVersion stands in when the build information carries no usable
+	// version, as happens with "go run", "go test" and unstamped builds.
+	unknownVersion = "devel"
+)
+
+// userAgent is resolved once: the build information doesn't change at runtime.
+var userAgent = buildUserAgent(buildInfo())
+
+// UserAgent returns the User-Agent naming this library and its version, e.g.
+//
+//	libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)
+//
+// The version comes from the build information of the running binary, and is
+// "devel" when there is none to be found.
+func UserAgent() string {
+	return userAgent
+}
+
+// userAgentForVersion formats the User-Agent for a Go module version.
+func userAgentForVersion(version string) string {
+	return fmt.Sprintf("%s/%s (+%s)", productName, normalizeVersion(version), projectURL)
+}
+
+// buildInfo returns the build information of the running binary, or nil when it
+// is unavailable.
+func buildInfo() *debug.BuildInfo {
+	info, _ := debug.ReadBuildInfo()
+
+	return info
+}
+
+// buildUserAgent formats the User-Agent for the version of this module recorded
+// in info.
+func buildUserAgent(info *debug.BuildInfo) string {
+	return userAgentForVersion(moduleVersion(info))
+}
+
+// moduleVersion digs the version of this module out of info. It answers an empty
+// string when info records none, which is the case for a binary built from a
+// list of files (as the released CLI is) and when info itself is nil.
+func moduleVersion(info *debug.BuildInfo) string {
+	if info == nil {
+		return ""
+	}
+
+	// The main module is this one when one of its own commands runs: its path is
+	// the module path for the module itself, and below it for a command.
+	if info.Main.Path == modulePath || strings.HasPrefix(info.Main.Path, modulePath+"/") {
+		return info.Main.Version
+	}
+
+	// The parser is a dependency when the library is embedded in another program.
+	for _, dep := range info.Deps {
+		if dep != nil && dep.Path == modulePath {
+			return dep.Version
+		}
+	}
+
+	return ""
+}
+
+// normalizeVersion turns a Go module version into what goes in the User-Agent:
+// a bare version without the "v" prefix, or unknownVersion when there is
+// nothing usable. A pseudo-version is kept as it is, since it identifies the
+// commit.
+func normalizeVersion(version string) string {
+	version = strings.TrimPrefix(version, "v")
+
+	// An unstamped main module reports "(devel)".
+	if version == "" || strings.HasPrefix(version, "(") {
+		return unknownVersion
+	}
+
+	return version
+}