POSIX MTC mirror config

cmd/mtc/mirror/internal/handler/handler.go

@@ -22,14 +22,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
-	"strconv"
 	"strings"
 
-	"log/slog"
-
 	"github.com/transparency-dev/tessera"
-	"github.com/transparency-dev/tessera/internal/parse"
+	"github.com/transparency-dev/tessera/internal/witness"
 )
 
 const (
@@ -55,75 +53,107 @@ func New(m Mirror) http.Handler {
 }
 
 func addCheckpoint(m Mirror) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		// SPEC: The mirror implements a [tlog-]witness's add-checkpoint endpoint.
-		//       MUST be a sequence of:
-		//         - an old size line,
-		//         - zero or more consistency proof lines,
-		//         - and an empty line,
-		//         - followed by a checkpoint.
+	const maxRequestBodyBytes = 64 << 10
 
-		reader := bufio.NewReader(r.Body)
-
-		// 1. Read old size line.
-		oldLine, err := reader.ReadString('\n')
+	return func(w http.ResponseWriter, r *http.Request) {
+		origin, oldSize, proof, cp, err := parseBody(http.MaxBytesReader(w, r.Body, maxRequestBodyBytes))
 		if err != nil {
-			http.Error(w, "missing old size", http.StatusBadRequest)
-			return
-		}
-		oldLine = strings.TrimSpace(oldLine)
-		if !strings.HasPrefix(oldLine, "old ") {
-			http.Error(w, "invalid old size line", http.StatusBadRequest)
+			slog.InfoContext(r.Context(), "Invalid witness request", slog.Any("error", err.Error()))
+			w.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		oldSize, err := strconv.ParseUint(strings.TrimPrefix(oldLine, "old "), 10, 64)
+
+		sc, body, contentType, err := handleUpdate(r.Context(), m, origin, oldSize, cp, proof)
 		if err != nil {
-			http.Error(w, "invalid old size", http.StatusBadRequest)
+			status := http.StatusInternalServerError
+			slog.InfoContext(r.Context(), "Witness update failed", slog.Any("error", err.Error()))
+			w.WriteHeader(status)
 			return
 		}
 
-		// 2. Read consistency proof lines until an empty line.
-		var proof [][]byte
-		for {
-			line, err := reader.ReadString('\n')
-			if err != nil {
-				http.Error(w, "unexpected EOF while reading proof", http.StatusBadRequest)
-				return
-			}
-			line = strings.TrimSpace(line)
-			if line == "" {
-				break
-			}
-			p, err := base64.StdEncoding.DecodeString(line)
-			if err != nil {
-				http.Error(w, "invalid proof line", http.StatusBadRequest)
-				return
+		if contentType != "" {
+			w.Header().Add("Content-Type", contentType)
+		}
+		w.WriteHeader(sc)
+		if len(body) > 0 {
+			if _, err := w.Write(body); err != nil {
+				slog.InfoContext(r.Context(), "Witness failed to write response", slog.Any("error", err.Error()))
 			}
-			proof = append(proof, p)
 		}
+	}
+}
 
-		// 3. Remaining data is the checkpoint.
-		cp, err := io.ReadAll(reader)
-		if err != nil {
-			http.Error(w, "failed to read checkpoint", http.StatusBadRequest)
-			return
+// handleUpdate submits the provided checkpoint to the witness and interprets any errors which may result.
+//
+// Returns an appropriate HTTP status code, response body, and Content Type representing the outcome.
+func handleUpdate(ctx context.Context, m Mirror, origin string, oldSize uint64, cp []byte, proof [][]byte) (int, []byte, string, error) {
+	sigs, trustedSize, updateErr := m.AddCheckpoint(ctx, origin, oldSize, proof, cp)
+	// Finally, handle any "soft" error from the update:
+	if updateErr != nil {
+		switch {
+		case errors.Is(updateErr, witness.ErrCheckpointStale):
+			return http.StatusConflict, fmt.Appendf(nil, "%d\n", trustedSize), "text/x.tlog.size", nil
+		case errors.Is(updateErr, witness.ErrUnknownLog):
+			return http.StatusNotFound, nil, "", nil
+		case errors.Is(updateErr, witness.ErrNoValidSignature):
+			return http.StatusForbidden, nil, "", nil
+		case errors.Is(updateErr, witness.ErrOldSizeInvalid):
+			return http.StatusBadRequest, nil, "", nil
+		case errors.Is(updateErr, witness.ErrInvalidProof):
+			return http.StatusUnprocessableEntity, nil, "", nil
+		case errors.Is(updateErr, witness.ErrRootMismatch):
+			return http.StatusConflict, nil, "", nil
+		default:
+			return http.StatusInternalServerError, nil, "", updateErr
 		}
+	}
+
+	return http.StatusOK, sigs, "", nil
+}
 
-		// 4. Extract the origin from the checkpoint and pass the request to the backend to take action.
-		origin, _, _, err := parse.CheckpointUnsafe(cp)
+// parseBody reads the incoming request and parses into constituent parts.
+//
+// The request body MUST be a sequence of
+// - a previous size line,
+// - zero or more consistency proof lines,
+// - and an empty line,
+// - followed by a [checkpoint][].
+func parseBody(r io.Reader) (string, uint64, [][]byte, []byte, error) {
+	b := bufio.NewReader(r)
+	sizeLine, _, err := b.ReadLine()
+	if err != nil {
+		return "", 0, nil, nil, err
+	}
+	var size uint64
+	if n, err := fmt.Sscanf(string(sizeLine), "old %d", &size); err != nil || n != 1 {
+		return "", 0, nil, nil, err
+	}
+	proof := [][]byte{}
+	for {
+		l, _, err := b.ReadLine()
 		if err != nil {
-			http.Error(w, fmt.Sprintf("invalid checkpoint: %v", err), http.StatusBadRequest)
-			return
+			return "", 0, nil, nil, err
 		}
-		if _, _, err := m.AddCheckpoint(r.Context(), origin, oldSize, proof, cp); err != nil {
-			slog.ErrorContext(r.Context(), "AddCheckpoint failed", slog.Any("error", err))
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
+		if len(l) == 0 {
+			break
 		}
-
-		w.WriteHeader(http.StatusOK)
-		// TODO(al): Maybe return a tlog-witness only cosignature here from a separate key?
+		hash, err := base64.StdEncoding.DecodeString(string(l))
+		if err != nil {
+			return "", 0, nil, nil, err
+		}
+		proof = append(proof, hash)
+	}
+	cp, err := io.ReadAll(b)
+	if err != nil {
+		return "", 0, nil, nil, err
 	}
+	s := strings.SplitN(string(cp), "\n", 2)
+	if len(s) != 2 {
+		return "", 0, nil, nil, err
+	}
+
+	origin := s[0]
+	return origin, size, proof, cp, nil
 }
 
 func addEntries(m Mirror) http.HandlerFunc {

cmd/mtc/mirror/internal/mirror/config.go

@@ -0,0 +1,75 @@
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mirror
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+
+	"golang.org/x/mod/sumdb/note"
+	f_note "github.com/transparency-dev/formats/note"
+)
+
+// Config is a placeholder structure for configuring mirrored logs.
+//
+// There will likely be a standard format at some point which we should try to adopt, but for now this is it.
+type Config struct {
+	Logs []LogConfig `json:"logs"`
+}
+
+// LogConfig represents a log.
+type LogConfig struct {
+	jsonLogConfig
+
+	Verifier note.Verifier `json:"-"`
+}
+
+type jsonLogConfig struct {
+	Vkey string `json:"vkey"`
+}
+
+// UnmarshalJSON handles the unmarshalling of LogConfig.
+// This is needed to parse the VKEY into a verifier.
+func (lc *LogConfig) UnmarshalJSON(data []byte) error {
+	r := &LogConfig{}
+	if err := json.Unmarshal(data, &r.jsonLogConfig); err != nil {
+		return err
+	}
+	if r.Vkey == "" {
+		return errors.New("vkey is required")
+	}
+	vk, err := f_note.NewVerifier(r.Vkey)
+	if err != nil {
+		return err
+	}
+	r.Verifier = vk
+	*lc = *r
+	return nil
+}
+
+func Load(path string) (Config, error) {
+	if path == "" {
+		return Config{}, errors.New("path is required")
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return Config{}, err
+	}
+	var cfg Config
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}

cmd/mtc/mirror/posix/main.go

@@ -19,23 +19,73 @@
 	"flag"
 	"net/http"
 	"os"
+	"path/filepath"
+	"net/url"
 
 	"log/slog"
 
+	"github.com/transparency-dev/formats/note"
+	"github.com/transparency-dev/tessera"
+	"github.com/transparency-dev/tessera/storage/posix"
 	"github.com/transparency-dev/tessera/cmd/mtc/mirror/internal/handler"
 	"github.com/transparency-dev/tessera/cmd/mtc/mirror/internal/mirror"
 )
 
 var (
-	listenAddr = flag.String("listen_addr", ":8080", "The address to listen on for HTTP requests.")
+	listenAddr  = flag.String("listen_addr", ":8080", "The address to listen on for HTTP requests.")
+	configPath  = flag.String("mirror_config", "", "Path to JSON config file describing mirror targets.")
+	privKeyFile = flag.String("private_key_path", "", "Path to file containing private key to use for cosigning checkpoints.")
+	storageRoot = flag.String("storage_root", "", "Path to directory to use for storing mirror data.")
+	slogLevel   = flag.Int("slog_level", int(slog.LevelInfo), "The cut-off threshold for structured logging. Default is 0 (INFO).")
 )
 
 func main() {
 	flag.Parse()
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
 	ctx := context.Background()
 
-	m := &mirror.Mirror{}
+	if *storageRoot == "" {
+		slog.ErrorContext(ctx, "storage_root must be set")
+		os.Exit(1)
+	}
+
+	if err := os.MkdirAll(*storageRoot, 0755); err != nil {
+		slog.ErrorContext(ctx, "Failed to create storage_root", slog.Any("error", err))
+		os.Exit(1)
+	}
+
+	cfg, err := mirror.Load(*configPath)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to load config", slog.Any("error", err))
+		os.Exit(1)
+	}
+
+	signer := signerFromFlags(ctx)
+
+	targets := make(map[string]mirror.Target)
+	for _, log := range cfg.Logs {
+		origin := log.Verifier.Name()
+		mirrorRoot := filepath.Join(*storageRoot, url.PathEscape(origin))
+		if err := os.MkdirAll(mirrorRoot, 0755); err != nil {
+			slog.ErrorContext(ctx, "Failed to create mirror root", slog.String("origin", origin), slog.Any("error", err))
+			os.Exit(1)
+		}
+		driver, err := posix.New(ctx, posix.Config{Path: mirrorRoot})
+		if err != nil {
+			slog.ErrorContext(ctx, "Failed to create driver", slog.String("origin", origin), slog.Any("error", err))
+			os.Exit(1)
+		}
+		t, err := tessera.NewMirrorTarget(ctx, driver, tessera.NewMirrorOptions().
+			WithLogVerifier(log.Verifier).
+			WithSigner(signer))
+		if err != nil {
+			slog.ErrorContext(ctx, "Failed to create mirror target", slog.String("origin", origin), slog.Any("error", err))
+			os.Exit(1)
+		}
+		targets[origin] = t
+	}
+
+	m := mirror.New(targets)
 	h := handler.New(m)
 
 	slog.InfoContext(ctx, "Starting mirror service", slog.String("addr", *listenAddr))
@@ -44,3 +94,22 @@
 		os.Exit(1)
 	}
 }
+
+// signerFromFlags creates a note.Signer for cosignature v1 signnatures from the command-line flags.
+func signerFromFlags(ctx context.Context) *note.Signer {
+	if *privKeyFile == "" {
+		slog.ErrorContext(ctx, "Missing private_key_path flag")
+		os.Exit(1)
+	}
+	pkRaw, err := os.ReadFile(*privKeyFile)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read private_key_path", slog.Any("error", err))
+		os.Exit(1)
+	}
+	signer, err := note.NewSignerForCosignatureV1(string(pkRaw))
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to create signer", slog.Any("error", err))
+		os.Exit(1)
+	}
+	return signer
+}

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/muesli/termenv v0.16.0
 	github.com/rivo/tview v0.42.0
-	github.com/transparency-dev/formats v0.1.0
+	github.com/transparency-dev/formats v0.1.2-0.20260608092312-797ea6fc4d76
 	github.com/transparency-dev/merkle v0.0.2
 	go.opentelemetry.io/contrib/detectors/aws/ec2/v2 v2.5.0
 	go.opentelemetry.io/contrib/detectors/aws/ecs v1.43.0
@@ -47,6 +47,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/monitoring v1.25.0 // indirect
 	cloud.google.com/go/trace v1.11.7 // indirect
+	filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect

go.sum

@@ -25,6 +25,8 @@ cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U
 cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
 filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
 filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
+filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4jvTs87qa5RqSN2U5FJU=
+filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 h1:BzsL0qE7LvtTEtXG7Dt5NS1EP0CQwI21HZfj9aGghhw=
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0/go.mod h1:I7kE2kM3qCr9QPT4cU4cCFYkEpVyVr16YOGUHzy+nR0=
@@ -240,8 +242,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/transparency-dev/formats v0.1.0 h1:oL0zUFuYUjg8AbtjPMnIRDmjbaHo5jCjEWU5yaNuz0g=
-github.com/transparency-dev/formats v0.1.0/go.mod h1:d2FibUOHfCMdCe/+/rbKt1IPLBbPTDfwj46kt541/mU=
+github.com/transparency-dev/formats v0.1.2-0.20260608092312-797ea6fc4d76 h1:KwVOf4Q/lQcL9FvMoED30VgPpc7wGsTudfNvitkMd1I=
+github.com/transparency-dev/formats v0.1.2-0.20260608092312-797ea6fc4d76/go.mod h1:qtZ8goRuJ8FTBG9c9+Bj0rn2rUG7eG/AUTkr+Aw3jFw=
 github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
 github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=

internal/witness/client.go

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package witness contains the implementation for sending out a checkpoint to witnesses
-// and retrieving sufficient signatures to satisfy a policy.
+// Package witness contains the implementations of a witness client, used to send out a checkpoint to witnesses
+// and retrieve sufficient signatures to satisfy a policy, and a witness service, used by the tlog-mirror implementation.
 package witness
 
 import (