fix(caddyconfig): custom global Caddy config is preserved on regeneration,warn about last load error in 'uc caddy config' (fixes #412)

api/pb/caddy.pb.go

@@ -27,10 +27,12 @@ type GetCaddyConfigResponse struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// The generated Caddyfile content.
+	// The saved Caddyfile content.
 	Caddyfile string `protobuf:"bytes,1,opt,name=caddyfile,proto3" json:"caddyfile,omitempty"`
 	// Timestamp when the config was last modified.
 	ModifiedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=modified_at,json=modifiedAt,proto3" json:"modified_at,omitempty"`
+	// Error from the latest unsuccessful reconciliation attempt.
+	LastReconciliationError string `protobuf:"bytes,3,opt,name=last_reconciliation_error,json=lastReconciliationError,proto3" json:"last_reconciliation_error,omitempty"`
 }
 
 func (x *GetCaddyConfigResponse) Reset() {
@@ -79,6 +81,13 @@ func (x *GetCaddyConfigResponse) GetModifiedAt() *timestamppb.Timestamp {
 	return nil
 }
 
+func (x *GetCaddyConfigResponse) GetLastReconciliationError() string {
+	if x != nil {
+		return x.LastReconciliationError
+	}
+	return ""
+}
+
 var File_api_pb_caddy_proto protoreflect.FileDescriptor
 
 var file_api_pb_caddy_proto_rawDesc = []byte{
@@ -87,22 +96,26 @@ var file_api_pb_caddy_proto_rawDesc = []byte{
 	0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79,
 	0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70,
 	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
-	0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x61,
+	0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xaf, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43,
-	0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01,
+	0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x12,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65,
-	0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02,
+	0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
+	0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
-	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
-	0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x32, 0x49, 0x0a, 0x05,
+	0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, 0x0a,
-	0x43, 0x61, 0x64, 0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66,
+	0x19, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61,
-	0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
+	0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
-	0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69,
+	0x52, 0x17, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61,
-	0x2e, 0x47, 0x65, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
+	0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x49, 0x0a, 0x05, 0x43, 0x61, 0x64,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75,
+	0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69,
+	0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
-	0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62,
+	0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
+	0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e,
+	0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

api/pb/caddy.proto

@@ -13,8 +13,10 @@ service Caddy {
 }
 
 message GetCaddyConfigResponse {
-  // The generated Caddyfile content.
+  // The saved Caddyfile content.
   string caddyfile = 1;
   // Timestamp when the config was last modified.
   google.protobuf.Timestamp modified_at = 2;
+  // Error from the latest unsuccessful reconciliation attempt.
+  string last_reconciliation_error = 3;
 }

cmd/uc/caddy/config.go

@@ -8,6 +8,7 @@ import (
 	"github.com/alecthomas/chroma/v2/quick"
 	"github.com/psviderski/uncloud/internal/cli"
 	"github.com/psviderski/uncloud/internal/cli/completion"
+	"github.com/psviderski/uncloud/internal/cli/tui"
 	"github.com/spf13/cobra"
 )
 
@@ -66,5 +67,10 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
 		}
 	}
 
+	if config.LastReconciliationError != "" {
+		tui.PrintWarning(fmt.Sprintf("last Caddy config load failed: %s\nShowing the last saved Caddyfile.",
+			config.LastReconciliationError))
+	}
+
 	return nil
 }

internal/machine/caddyconfig/caddyfile.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"cmp"
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"maps"
@@ -57,14 +58,12 @@ https://{{$hostname}} {
 	log
 }{{end}}
 `
-	caddyfileUnavailabeFooter = `# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
+	bootstrapMarker       = "# Uncloud bootstrap for Caddy container "
-#       (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
+	legacyBootstrapMarker = "# NOTE: User-defined configs for services were skipped because Caddy is not running"
-#       generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
-#       and its logs for more details (uc logs caddy).
-`
 )
 
 // CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy.
+// It combines generated routes from published ports with user-defined Caddyfile snippets from service specs (x-caddy).
 type CaddyfileGenerator struct {
 	// machineID is the unique identifier of the machine where the controller is running.
 	machineID string
@@ -74,11 +73,18 @@ type CaddyfileGenerator struct {
 	log         *slog.Logger
 }
 
-// CaddyfileValidator is an interface for validating Caddyfile configurations.
+// CaddyfileValidator checks a candidate Caddyfile without loading it. Validate returns InvalidCaddyfileError only
+// when the candidate is known to be invalid. Other errors mean the check could not be completed. A successful check
+// does not guarantee that the configuration will load.
 type CaddyfileValidator interface {
 	Validate(ctx context.Context, caddyfile string) error
 }
 
+// InvalidCaddyfileError means validation completed and the candidate Caddyfile was rejected.
+type InvalidCaddyfileError struct{ Message string }
+
+func (e *InvalidCaddyfileError) Error() string { return e.Message }
+
 func NewCaddyfileGenerator(
 	machineID, machineName string, validator CaddyfileValidator, log *slog.Logger,
 ) *CaddyfileGenerator {
@@ -93,35 +99,41 @@ func NewCaddyfileGenerator(
 	}
 }
 
-// Generate creates a Caddyfile configuration based on the provided service containers.
+// Generate creates a Caddyfile for the local Caddy container from the supplied healthy application containers.
-// The Caddyfile is generated from the service ports of the healthy containers.
+// Application service ports provide the generated sites. The Caddy container's custom Caddy config (x-caddy), if
-// If a 'caddy' service container is running on this machine and defines a custom Caddy config (x-caddy) in its service
+// defined, provides the global block even when that container is unhealthy. When application custom Caddy configs
-// spec, it will be validated and prepended to the generated Caddyfile. Custom Caddy configs (x-caddy) defined in other
+// are included, the newest container for each service provides its config.
-// service specs are validated and appended to the generated Caddyfile. Invalid configs are logged and skipped to ensure
-// the generated Caddyfile remains valid.
 //
-// The final Caddyfile structure includes:
+// The resulting Caddyfile has this structure:
 //
-//	[caddy x-caddy (global config)]
+//	[caddy custom Caddy config (global)]
-//	[generated Caddyfile from all service ports]
+//	[generated sites from application service ports]
-//	[service-a x-caddy]
+//	[service-a custom Caddy config]
 //	...
-//	[service-z x-caddy]
+//	[service-z custom Caddy config]
 //
-// If includeCustom is false, custom Caddy configs (x-caddy) are not included in the generated Caddyfile.
+// Bootstrap mode keeps the global custom Caddy config and generated sites, but omits application custom Caddy configs.
+// It skips validation because Caddy may not be running yet. In this case, Caddy validates the config when it starts.
+// Global template errors still stop generation. In full mode, invalid globals stop generation, while invalid
+// application custom Caddy configs are logged and skipped.
 func (g *CaddyfileGenerator) Generate(
-	ctx context.Context, records []store.ContainerRecord, includeCustom bool,
+	ctx context.Context,
+	caddyCtr api.ServiceContainer,
+	records []store.ContainerRecord,
+	bootstrap bool,
 ) (string, error) {
+	records = slices.Clone(records)
 	// Sort records by local machine first, then by service name and creation time. Placing containers on the local
 	// machine first lets user-defined Caddy configs pair this ordering with the "first" lb_policy to always send
 	// traffic to the same-host replica (skipping the cross-machine hop) and only fall back to remote upstreams when
 	// the local one is unhealthy.
-	// The service name and creation time tiebreakers keep the generated Caddyfile stable across regenerations.
+	// The service name, creation time, and container ID tiebreakers keep the file stable across regenerations.
 	slices.SortStableFunc(records, func(a, b store.ContainerRecord) int {
 		return cmp.Or(
 			g.localMachineRank(a.MachineID)-g.localMachineRank(b.MachineID),
 			strings.Compare(a.Container.ServiceName(), b.Container.ServiceName()),
 			a.Container.CreatedTime().Compare(b.Container.CreatedTime()),
+			strings.Compare(a.Container.ID, b.Container.ID),
 		)
 	})
 
@@ -136,26 +148,11 @@ func (g *CaddyfileGenerator) Generate(
 	}
 
 	caddyfileHeader := fmt.Sprintf(caddyfileHeaderFmt, g.machineName, time.Now().UTC().Format(time.RFC3339))
-	if !includeCustom {
-		return fmt.Sprintf("%s\n%s\n%s", caddyfileHeader, caddyfile, caddyfileUnavailabeFooter), nil
-	}
-
 	upstreams := serviceUpstreams(containers)
 	// Track validation errors for reporting.
 	var configErrors []string
 
-	// Find the 'caddy' service container on this machine. Use the most recent one if multiple exist.
+	if caddyCtr.ServiceSpec.CaddyConfig() != "" {
-	var caddyCtr *api.ServiceContainer
-	for _, cr := range records {
-		if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName &&
-			(caddyCtr == nil || cr.Container.CreatedTime().Compare(caddyCtr.CreatedTime()) > 0) {
-			caddyCtr = &cr.Container
-		}
-	}
-
-	// If the caddy container is running on this machine and has a custom Caddy config (global),
-	// prepend it to the generated Caddyfile and validate it.
-	if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" {
 		// Render the custom global Caddy config as a Go template with the upstreams.
 		tmplCtx := templateContext{
 			Name:      caddyCtr.ServiceName(),
@@ -163,23 +160,23 @@ func (g *CaddyfileGenerator) Generate(
 		}
 		renderedConfig, err := renderCaddyfile(tmplCtx, caddyCtr.ServiceSpec.CaddyConfig())
 		if err != nil {
-			g.log.Error("Failed to render template directives in user-defined global Caddy config, skipping it.",
+			return "", fmt.Errorf(
-				"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
+				"render template directives in user-defined global Caddy config from Caddy container %s: %w",
-			configErrors = append(configErrors,
+				caddyCtr.ShortID(), err)
-				fmt.Sprintf("service '%s': failed to render template: %v", caddyCtr.ServiceName(), err))
+		}
-		} else {
 		caddyfileCandidate := fmt.Sprintf("# User-defined global config from service '%s'.\n%s\n\n%s",
 			caddyCtr.ServiceName(), renderedConfig, caddyfile)
-
+		if !bootstrap && g.validator != nil {
 			if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
-				g.log.Error("User-defined global Caddy config is invalid, skipping it.",
+				return "", fmt.Errorf("validate user-defined global Caddy config from Caddy container %s: %w",
-					"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
+					caddyCtr.ShortID(), err)
-				configErrors = append(configErrors,
+			}
-					fmt.Sprintf("service '%s': validation failed: %v", caddyCtr.ServiceName(), err))
+		}
-			} else {
 		caddyfile = caddyfileCandidate
 	}
-		}
+
+	if bootstrap {
+		return caddyfileHeader + bootstrapMarker + caddyCtr.ID + "\n\n" + caddyfile, nil
 	}
 
 	// There could be multiple service containers for the same service with different custom Caddy configs, for example,
@@ -200,7 +197,7 @@ func (g *CaddyfileGenerator) Generate(
 	// Append a custom Caddy config for each service to the Caddyfile and validate it. If the config for a service
 	// is invalid, skip it but continue processing other services to ensure the Caddyfile remains valid.
 	for _, serviceName := range sortedServiceNames {
-		// Skip the caddy container as we already processed it.
+		// Skip the caddy container as we already processed it as the global config.
 		if serviceName == CaddyServiceName {
 			continue
 		}
@@ -226,16 +223,22 @@ func (g *CaddyfileGenerator) Generate(
 
 		caddyfileCandidate := fmt.Sprintf("%s\n# User-defined config for service '%s'.\n%s\n",
 			caddyfile, serviceName, renderedConfig)
+		if g.validator != nil {
 			if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
+				if _, ok := errors.AsType[*InvalidCaddyfileError](err); !ok {
+					return "", fmt.Errorf("validate Caddy config for service '%s': %w", serviceName, err)
+				}
 				g.log.Error("User-defined Caddy config for service is invalid, skipping it.",
 					"service", serviceName, "err", err)
-			configErrors = append(configErrors, fmt.Sprintf("service '%s': validation failed: %v", serviceName, err))
+				configErrors = append(configErrors,
-		} else {
+					fmt.Sprintf("service '%s': validation failed: %v", serviceName, err))
+				continue
+			}
+		}
 		caddyfile = caddyfileCandidate
 	}
-	}
 
-	// Append error summary as comment if there were any invalid configs.
+	// Keep skipped-snippet errors in the saved file so an operator can see why routes are missing after a restart.
 	if len(configErrors) > 0 {
 		var errorsComment strings.Builder
 		errorsComment.WriteString("# Skipped invalid user-defined configs:\n")
@@ -249,8 +252,7 @@ func (g *CaddyfileGenerator) Generate(
 	return caddyfileHeader + "\n" + caddyfile, nil
 }
 
-// localMachineRank returns 0 if the given machineID matches the local machine and 1 otherwise.
+// localMachineRank is a sorting function for containers that puts running on the local machine first.
-// Useful for sorting containers running locally first.
 func (g *CaddyfileGenerator) localMachineRank(machineID string) int {
 	if g.machineID == machineID {
 		return 0

internal/machine/caddyconfig/caddyfile_test.go

@@ -208,10 +208,11 @@ http://app.example.com {
 	ctx := context.Background()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			// Validator is not expected to be called in these tests.
+			validator := NewMockCaddyfileValidator(t)
-			generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil)
+			generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
 
-			config, err := generator.Generate(ctx, tt.containers, true)
+			caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.1", "", "", time.Now()).Container
+			config, err := generator.Generate(ctx, caddyCtr, tt.containers, false)
 
 			if tt.wantErr {
 				assert.Error(t, err)
@@ -332,7 +333,7 @@ bad.template.com {
 `,
 		},
 		{
-			name: "caddy service with invalid global config is skipped",
+			name: "caddy service with invalid global config aborts",
 			containers: []store.ContainerRecord{
 				newContainerRecordWithCaddyConfig(
 					"caddy",
@@ -345,10 +346,7 @@ localhost {
 					time.Now(),
 				),
 			},
-			want: testCaddyfileHeader + `
+			wantErr: true,
-# Skipped invalid user-defined configs:
-# - service 'caddy': validation failed: invalid config detected
-`,
 		},
 		{
 			name: "caddy service on different machine is ignored",
@@ -786,7 +784,7 @@ web-v2.example.com {
 `,
 		},
 		{
-			name: "multiple errors: invalid global, template error, and validation error",
+			name: "invalid global aborts before application configs",
 			containers: []store.ContainerRecord{
 				newContainerRecordWithCaddyConfig(
 					"caddy",
@@ -827,17 +825,7 @@ invalid.example.com {
 					time.Now(),
 				),
 			},
-			want: testCaddyfileHeader + `
+			wantErr: true,
-# User-defined config for service 'valid'.
-valid.example.com {
-	respond "Valid config"
-}
-
-# Skipped invalid user-defined configs:
-# - service 'caddy': validation failed: invalid config detected
-# - service 'broken-template': failed to render template: parse config as Go template: template: Caddyfile:2: unterminated quoted string
-# - service 'invalid': validation failed: invalid config detected
-`,
 		},
 	}
 
@@ -846,7 +834,7 @@ valid.example.com {
 	validator.EXPECT().Validate(mock.Anything, mock.Anything).RunAndReturn(
 		func(ctx context.Context, caddyfile string) error {
 			if strings.Contains(caddyfile, "# test:invalid") {
-				return errors.New("invalid config detected")
+				return &InvalidCaddyfileError{Message: "invalid config detected"}
 			}
 			return nil
 		})
@@ -855,7 +843,8 @@ valid.example.com {
 		t.Run(tt.name, func(t *testing.T) {
 			generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
 
-			config, err := generator.Generate(ctx, tt.containers, true)
+			caddyCtr := caddyContainerFromTestContainers(tt.containers)
+			config, err := generator.Generate(ctx, caddyCtr, tt.containers, false)
 
 			if tt.wantErr {
 				assert.Error(t, err)
@@ -868,6 +857,167 @@ valid.example.com {
 	}
 }
 
+func TestCaddyfileGeneratorBootstrap(t *testing.T) {
+	// Bootstrap keeps the local Caddy container's globals and generated routes, but omits application custom configs.
+	tests := []struct {
+		name       string
+		containers []store.ContainerRecord
+		want       string
+	}{
+		{
+			name: "global config retained and application custom configs skipped",
+			containers: []store.ContainerRecord{
+				newContainerRecordWithCaddyConfig(
+					"caddy",
+					"10.210.0.1",
+					`# Global config
+{
+	global directive
+}`,
+					"test-machine-id",
+					time.Now(),
+				),
+				newContainerRecordWithCaddyConfig(
+					"web",
+					"10.210.0.2",
+					`web.example.com {
+	reverse_proxy web:3000
+}`,
+					"test-machine-id",
+					time.Now(),
+				),
+				newContainerRecordWithPorts(
+					"api",
+					"10.210.0.3",
+					[]string{"api.example.com:8080/http"},
+					"test-machine-id",
+				),
+			},
+			want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", `
+# Uncloud bootstrap for Caddy container caddy-10.210.0.1
+
+# User-defined global config from service 'caddy'.
+# Global config
+{
+	global directive
+}
+
+# Health check endpoint`, 1) + `
+# Sites generated from service ports.
+
+http://api.example.com {
+	reverse_proxy 10.210.0.3:8080 {
+		import common_proxy
+	}
+	log
+}
+`,
+		},
+		{
+			name: "no containers with x-caddy configs",
+			containers: []store.ContainerRecord{
+				newContainerRecordWithPorts(
+					"api",
+					"10.210.0.3",
+					[]string{"api.example.com:8080/http"},
+					"test-machine-id",
+				),
+			},
+			want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", `
+# Uncloud bootstrap for Caddy container caddy-10.210.0.1
+
+# Health check endpoint`, 1) + `
+# Sites generated from service ports.
+
+http://api.example.com {
+	reverse_proxy 10.210.0.3:8080 {
+		import common_proxy
+	}
+	log
+}
+`,
+		},
+	}
+
+	ctx := context.Background()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			validator := NewMockCaddyfileValidator(t)
+			generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
+
+			caddyContainer := caddyContainerFromTestContainers(tt.containers)
+			config, err := generator.Generate(ctx, caddyContainer, tt.containers, true)
+			require.NoError(t, err)
+
+			assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match")
+		})
+	}
+}
+
+func TestCaddyfileGenerator_ValidatorTransportErrorAbortsGeneration(t *testing.T) {
+	validator := NewMockCaddyfileValidator(t)
+	validator.EXPECT().Validate(mock.Anything, mock.Anything).Return(errors.New("socket disappeared"))
+
+	generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
+	app := newContainerRecordWithCaddyConfig(
+		"web",
+		"10.210.0.2",
+		"web.example.com { respond ok }",
+		"test-machine-id",
+		time.Now(),
+	)
+
+	_, err := generator.Generate(context.Background(), caddyContainerFromTestContainers(nil),
+		[]store.ContainerRecord{app}, false)
+	require.ErrorContains(t, err, "socket disappeared")
+}
+
+func newContainer(ip string, ports ...string) api.ServiceContainer {
+	portsLabel := strings.Join(ports, ",")
+	return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
+		ContainerJSONBase: &container.ContainerJSONBase{
+			State: &container.State{
+				Running: true,
+			},
+		},
+		NetworkSettings: &container.NetworkSettings{
+			Networks: map[string]*network.EndpointSettings{
+				docker.NetworkName: {
+					IPAddress: ip,
+				},
+			},
+		},
+		Config: &container.Config{
+			Labels: map[string]string{
+				api.LabelServicePorts: portsLabel,
+			},
+		},
+	}}}
+}
+
+func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
+	portsLabel := strings.Join(ports, ",")
+	return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
+		ContainerJSONBase: &container.ContainerJSONBase{
+			State: &container.State{
+				Running: true,
+			},
+		},
+		NetworkSettings: &container.NetworkSettings{
+			Networks: map[string]*network.EndpointSettings{
+				"other-network": {
+					IPAddress: "172.17.0.2",
+				},
+			},
+		},
+		Config: &container.Config{
+			Labels: map[string]string{
+				api.LabelServicePorts: portsLabel,
+			},
+		},
+	}}}
+}
+
 func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord {
 	return store.ContainerRecord{
 		Container: ctr,
@@ -911,100 +1061,6 @@ func newContainerRecordWithCaddyConfig(serviceName, ip, caddyConfig, machineID s
 	}
 }
 
-func TestCaddyfileGeneratorWithoutCustomConfigs(t *testing.T) {
-	// Test that when includeCustom is false (Caddy not available), x-caddy configs are skipped.
-	tests := []struct {
-		name       string
-		containers []store.ContainerRecord
-		want       string
-	}{
-		{
-			name: "x-caddy configs are skipped",
-			containers: []store.ContainerRecord{
-				newContainerRecordWithCaddyConfig(
-					"caddy",
-					"10.210.0.1",
-					`# Global config
-{
-	global directive
-}`,
-					"test-machine-id",
-					time.Now(),
-				),
-				newContainerRecordWithCaddyConfig(
-					"web",
-					"10.210.0.2",
-					`web.example.com {
-	reverse_proxy web:3000
-}`,
-					"test-machine-id",
-					time.Now(),
-				),
-				newContainerRecordWithPorts(
-					"api",
-					"10.210.0.3",
-					[]string{"api.example.com:8080/http"},
-					"test-machine-id",
-				),
-			},
-			want: testCaddyfileHeader + `
-# Sites generated from service ports.
-
-http://api.example.com {
-	reverse_proxy 10.210.0.3:8080 {
-		import common_proxy
-	}
-	log
-}
-
-# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
-#       (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
-#       generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
-#       and its logs for more details (uc logs caddy).
-`,
-		},
-		{
-			name: "no containers with x-caddy configs",
-			containers: []store.ContainerRecord{
-				newContainerRecordWithPorts(
-					"api",
-					"10.210.0.3",
-					[]string{"api.example.com:8080/http"},
-					"test-machine-id",
-				),
-			},
-			want: testCaddyfileHeader + `
-# Sites generated from service ports.
-
-http://api.example.com {
-	reverse_proxy 10.210.0.3:8080 {
-		import common_proxy
-	}
-	log
-}
-
-# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
-#       (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
-#       generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
-#       and its logs for more details (uc logs caddy).
-`,
-		},
-	}
-
-	ctx := context.Background()
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Validator is not expected to be called in these tests.
-			generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil)
-
-			config, err := generator.Generate(ctx, tt.containers, false)
-			require.NoError(t, err)
-
-			assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match")
-		})
-	}
-}
-
 func newContainerRecordWithPorts(serviceName, ip string, ports []string, machineID string) store.ContainerRecord {
 	portsLabel := strings.Join(ports, ",")
 	return store.ContainerRecord{
@@ -1037,3 +1093,18 @@ func newContainerRecordWithPorts(serviceName, ip string, ports []string, machine
 		MachineID: machineID,
 	}
 }
+
+// caddyContainerFromTestContainers returns the local Caddy container from the provided test containers,
+// or creates a default one if not found.
+func caddyContainerFromTestContainers(records []store.ContainerRecord) api.ServiceContainer {
+	if caddyCtr := selectLocalCaddyContainer(records, "test-machine-id"); caddyCtr != nil {
+		return *caddyCtr
+	}
+	return newContainerRecordWithCaddyConfig(
+		"caddy",
+		"10.210.0.1",
+		"",
+		"test-machine-id",
+		time.Now(),
+	).Container
+}

internal/machine/caddyconfig/client.go

@@ -3,7 +3,6 @@ package caddyconfig
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -26,15 +25,15 @@ func NewCaddyAdminClient(socketPath string) *CaddyAdminClient {
 		client: &http.Client{
 			Timeout: 5 * time.Second,
 			Transport: &http.Transport{
-				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
-					return net.Dial("unix", socketPath)
+					return (&net.Dialer{}).DialContext(ctx, "unix", socketPath)
 				},
 			},
 		},
 	}
 }
 
-// IsAvailable checks if the local Caddy instance is listening on the admin socket.
+// IsAvailable checks whether a Caddy process accepts connections at the admin socket.
 func (c *CaddyAdminClient) IsAvailable() bool {
 	conn, err := net.DialTimeout("unix", c.socketPath, 1*time.Second)
 	// A stale socket file left over from a crashed Caddy container returns ECONNREFUSED so this is correctly handled
@@ -80,15 +79,16 @@ func (c *CaddyAdminClient) Adapt(ctx context.Context, caddyfile string) (string,
 	// If the response is a 400 Bad Request, try to parse the error message from it.
 	if resp.StatusCode == http.StatusBadRequest {
 		var apiError caddy.APIError
-		if err = json.Unmarshal(body, &apiError); err == nil {
+		if err = json.Unmarshal(body, &apiError); err == nil && apiError.Message != "" {
-			return "", errors.New(apiError.Message)
+			return "", &InvalidCaddyfileError{Message: apiError.Message}
 		}
+		return "", &InvalidCaddyfileError{Message: string(body)}
 	}
 
-	return "", errors.New(string(body))
+	return "", fmt.Errorf("adapt request failed: HTTP %d: %s", resp.StatusCode, string(body))
 }
 
-// Load loads a Caddyfile configuration into the Caddy instance running on the machine.
+// Load loads a Caddyfile configuration into the Caddy instance.
 // Due to a Caddy bug (https://github.com/caddyserver/caddy/issues/7246), we first adapt the Caddyfile to JSON
 // and then load the JSON config to get proper error handling.
 func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error {
@@ -96,7 +96,11 @@ func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error {
 	if err != nil {
 		return fmt.Errorf("adapt Caddyfile to JSON config: %w", err)
 	}
+	return c.LoadJSON(ctx, jsonConfig)
+}
 
+// LoadJSON loads a JSON configuration into the Caddy instance.
+func (c *CaddyAdminClient) LoadJSON(ctx context.Context, jsonConfig string) error {
 	req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/load", strings.NewReader(jsonConfig))
 	if err != nil {
 		return fmt.Errorf("create load request: %w", err)

internal/machine/caddyconfig/client_test.go

@@ -0,0 +1,54 @@
+package caddyconfig
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testAdminServer(t *testing.T, handler http.HandlerFunc) string {
+	t.Helper()
+	// macOS limits Unix socket paths to 104 bytes. t.TempDir can exceed that.
+	dir, err := os.MkdirTemp("/tmp", "uc-caddy-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(dir)
+	})
+
+	socketPath := filepath.Join(dir, "admin.sock")
+	listener, err := net.Listen("unix", socketPath)
+	require.NoError(t, err)
+	server := httptest.NewUnstartedServer(handler)
+	_ = server.Listener.Close()
+	server.Listener = listener
+	server.Start()
+	t.Cleanup(server.Close)
+
+	return socketPath
+}
+
+func TestCaddyAdminClient_Validate(t *testing.T) {
+	var status atomic.Int64
+	status.Store(http.StatusBadRequest)
+	socket := testAdminServer(t, func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(int(status.Load()))
+		_, _ = w.Write([]byte(`{"message":"invalid config"}`))
+	})
+	client := NewCaddyAdminClient(socket)
+	err := client.Validate(context.Background(), "invalid")
+	var invalid *InvalidCaddyfileError
+	assert.ErrorAs(t, err, &invalid)
+
+	status.Store(http.StatusInternalServerError)
+	err = client.Validate(context.Background(), "invalid")
+	assert.Error(t, err)
+	assert.NotErrorAs(t, err, &invalid)
+}

internal/machine/caddyconfig/controller.go

@@ -2,7 +2,7 @@ package caddyconfig
 
 import (
 	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/netip"
@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/psviderski/uncloud/internal/fs"
 	"github.com/psviderski/uncloud/internal/machine/store"
@@ -20,27 +21,40 @@ const (
 	CaddyServiceName = "caddy"
 	CaddyGroup       = "uncloud"
 	VerifyPath       = "/.uncloud-verify"
+	// periodicReconciliationInterval is the interval at which the controller reconciles the Caddyfile
+	// even if no container changes are observed.
+	periodicReconciliationInterval = 30 * time.Second
 )
 
-// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
+// Controller keeps the local Caddy reverse proxy in sync with container state from the cluster store. It writes a
-// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
+// Caddyfile that routes external traffic to healthy service containers across the internal network.
-// network.
+//
+// Current Caddy deployments share one Caddyfile and one admin socket per machine. The controller must preserve a
+// saved full Caddyfile when Caddy is unavailable because that file may be needed to restart or roll back a container.
+// It writes a reduced bootstrap config only when no full file exists, including when an older offline file must be
+// replaced. The shared layout cannot give overlapping Caddy revisions separate configurations.
 type Controller struct {
 	machineID     string
 	caddyfilePath string
+	service       *Service
 	generator     *CaddyfileGenerator
 	client        *CaddyAdminClient
 	store         *store.Store
 	log           *slog.Logger
-	// lastFingerprint caches the fingerprint of the containers used to generate the latest successfully loaded
+	// Tests use their own group so they can exercise real file publication without the daemon's uncloud group.
-	// Caddyfile. nil means it hasn't been loaded yet or the last load failed.
+	fileGroup string
+	// lastFingerprint records the healthy application-container inputs used for the last full Caddyfile that was
+	// successfully loaded into Caddy and saved to disk. It excludes the Caddy container, whose identity, start time,
+	// and global config are tracked separately below.
 	lastFingerprint []containerFingerprint
-	// lastCaddyfile caches the last generated Caddyfile.
+	lastCaddyCtrID  string
-	lastCaddyfile string
+	lastStartedAt   string
+	lastGlobal      string
+	lastSavedBody   string
 }
 
-// containerFingerprint is the subset of container data that the Caddyfile generator depends on.
+// containerFingerprint contains container identity and routing inputs that can change the generated Caddyfile.
-// Comparing fingerprints lets the controller skip no-op regenerations.
+// It excludes incidental Docker state so unrelated container updates do not cause a Caddy reload.
 type containerFingerprint struct {
 	ID          string
 	IP          netip.Addr
@@ -56,7 +70,8 @@ func (f containerFingerprint) Equal(other containerFingerprint) bool {
 		f.CaddyConfig == other.CaddyConfig
 }
 
-func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) {
+func NewController(machineID string, service *Service, adminSock string, store *store.Store) (*Controller, error) {
+	configDir := service.configDir
 	if err := os.MkdirAll(configDir, 0o750); err != nil {
 		return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err)
 	}
@@ -67,16 +82,21 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) (
 	log := slog.With("component", "caddy-controller")
 	client := NewCaddyAdminClient(adminSock)
 
-	// generator is initialised by Run() once the machine name is resolved from the store.
+	// The generator is initialised by Run() after resolving the machine name from the store.
 	return &Controller{
 		machineID:     machineID,
 		caddyfilePath: filepath.Join(configDir, "Caddyfile"),
+		service:       service,
 		client:        client,
 		store:         store,
 		log:           log,
+		fileGroup:     CaddyGroup,
 	}, nil
 }
 
+// Run reconciles on container changes and every periodicReconciliationInterval. Failed work is retried every second,
+// but repeated retry failures are logged only during the periodic check.
+// The periodic check also covers missed events and outstanding failures after the daemon restarts.
 func (c *Controller) Run(ctx context.Context) error {
 	// Default the machine name to the machine ID so the Caddyfile header still carries a stable identifier if
 	// the store lookup fails.
@@ -95,13 +115,34 @@ func (c *Controller) Run(ctx context.Context) error {
 	}
 	c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
 
-	containers = filterHealthyContainers(containers)
+	var retryC <-chan time.Time
-	c.generateAndLoadCaddyfile(ctx, containers)
+	periodic := time.NewTicker(periodicReconciliationInterval)
+	defer periodic.Stop()
 
-	// TODO: left for backward compatibility, remove later.
+	handleResult := func(err error, logFailure bool) {
-	if err = c.generateJSONConfig(containers); err != nil {
+		c.service.setReconciliationResult(err)
-		c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
+		if err == nil {
+			retryC = nil
+			return
 		}
+		if logFailure {
+			c.log.Error("Failed to reconcile Caddy configuration, will retry.", "err", err)
+		}
+		retryC = time.After(time.Second)
+	}
+
+	reconcile := func(logFailure bool) {
+		ctrs, err := c.store.ListContainers(ctx, store.ListOptions{})
+		if err != nil {
+			err = fmt.Errorf("list containers: %w", err)
+		} else {
+			err = c.generateAndLoadCaddyfile(ctx, ctrs)
+		}
+		handleResult(err, logFailure)
+	}
+
+	// The subscription supplies an initial snapshot, so the first attempt need not wait for a change event.
+	handleResult(c.generateAndLoadCaddyfile(ctx, containers), true)
 
 	for {
 		select {
@@ -111,25 +152,18 @@ func (c *Controller) Run(ctx context.Context) error {
 			}
 			c.log.Debug("Cluster containers changed, regenerating Caddy configuration.")
 
-			containers, err = c.store.ListContainers(ctx, store.ListOptions{})
+			reconcile(true)
-			if err != nil {
+		case <-periodic.C:
-				c.log.Error("Failed to list containers.", "err", err)
+			reconcile(true)
-				continue
+		case <-retryC:
-			}
+			reconcile(false)
-			containers = filterHealthyContainers(containers)
-			c.generateAndLoadCaddyfile(ctx, containers)
-
-			// TODO: left for backward compatibility, remove later.
-			if err = c.generateJSONConfig(containers); err != nil {
-				c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
-			}
 		case <-ctx.Done():
 			return nil
 		}
 	}
 }
 
-// filterHealthyContainers filters out unhealthy and hook containers.
+// filterHealthyContainers filters out unhealthy, hook, and caddy containers.
 // TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined
 // by the cluster membership state of the machine that the container is running on. Implement machine membership
 // check using Corrossion Admin client.
@@ -139,6 +173,9 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain
 		if cr.Container.IsHook() {
 			continue
 		}
+		if cr.Container.ServiceName() == CaddyServiceName {
+			continue
+		}
 		if cr.Container.Healthy() {
 			healthy = append(healthy, cr)
 		}
@@ -146,65 +183,96 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain
 	return healthy
 }
 
-// generateAndLoadCaddyfile regenerates the Caddyfile from the given containers and loads it into the local Caddy
+// generateAndLoadCaddyfile reconciles the shared Caddyfile for the selected local Caddy container. A full candidate
-// if available.
+// reaches disk only after Caddy accepts it. Without an admin endpoint, the controller keeps a saved full file intact
-func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) {
+// and writes a bootstrap config only when the file is absent or already a bootstrap.
-	// Check if Caddy is available before attempting to generate and load config.
+func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) error {
-	caddyAvailable := c.client.IsAvailable()
+	caddyCtr := selectLocalCaddyContainer(containers, c.machineID)
-
+	if caddyCtr == nil {
-	// Skip regeneration when Caddy is available and the containers since the last successful load haven't changed.
+		// Caddy is not running locally which means there is no reliable source for the global config, skipping.
-	// When Caddy is unavailable we still regenerate to keep the Caddyfile on disk updated.
+		return nil
-	fingerprint := fingerprintContainers(containers)
-	if caddyAvailable && slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) {
-		c.log.Debug("Caddy configuration is unchanged.", "path", c.caddyfilePath)
-		return
 	}
 
-	caddyfile, err := c.generator.Generate(ctx, containers, caddyAvailable)
+	saved, readErr := os.ReadFile(c.caddyfilePath)
+	if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
+		return fmt.Errorf("read saved Caddyfile: %w", readErr)
+	}
+	haveSaved := readErr == nil
+
+	healthyCtrs := filterHealthyContainers(containers)
+	fingerprint := fingerprintContainers(healthyCtrs)
+
+	if !caddyCtr.State.Running || !c.client.IsAvailable() {
+		if haveSaved && !isBootstrapCaddyfile(string(saved)) {
+			// Caddy may need this file to restart or roll back. A hosts-only replacement could drop
+			// storage or other global settings while the admin API is unavailable.
+			if caddyCtr.State.Running {
+				return fmt.Errorf("caddy admin socket unavailable, preserving saved Caddyfile")
+			}
+			return nil
+		}
+
+		bootstrap, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, true)
 		if err != nil {
-		c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
+			return fmt.Errorf("generate Caddy bootstrap config: %w", err)
-		return
+		}
+		if err = c.writeCaddyfileIfChanged(bootstrap); err != nil {
+			return fmt.Errorf("save Caddy bootstrap config: %w", err)
+		}
+		if caddyCtr.State.Running {
+			return fmt.Errorf("caddy admin socket unavailable, bootstrap config saved")
+		}
+		return nil
 	}
 
-	if !caddyAvailable {
+	if haveSaved &&
-		// Caddy is not running so the generated Caddyfile should not include user-defined configs thus must be valid.
+		caddyfileBody(string(saved)) == c.lastSavedBody &&
-		// It's safe to write the config to disk so that when Caddy is deployed on this machine, it can pick it up.
+		!isBootstrapCaddyfile(string(saved)) &&
-		if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
+		c.lastCaddyCtrID == caddyCtr.ID &&
-			c.log.Error("Failed to write Caddyfile to disk.", "err", err)
+		c.lastStartedAt == caddyCtr.State.StartedAt &&
-			return
+		c.lastGlobal == caddyCtr.ServiceSpec.CaddyConfig() &&
-		}
+		slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) {
-		c.log.Debug("Caddy is not running on this machine, skipping configuration load.", "path", c.caddyfilePath)
+		// No changes to the inputs that affect the generated Caddyfile, so no reload is needed.
-		return
+		return nil
 	}
 
-	// Caddy is available, try to load the config which may fail if the config is invalid. Generally, a config can
+	caddyfile, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, false)
-	// pass the adaptation/validation step but still fail to load, for example, if it references resources that are
+	if err != nil {
-	// not available.
+		return fmt.Errorf("generate Caddyfile: %w", err)
+	}
+	// Try to load the config which may fail if the config is invalid. Generally, a config can pass
+	// the adaptation/validation step but still fail to load, for example, if it references resources
+	// that are not available.
 	if err = c.client.Load(ctx, caddyfile); err != nil {
-		c.log.Error("Failed to load new Caddy configuration into local Caddy instance.",
+		return fmt.Errorf("load Caddyfile: %w", err)
-			"err", err, "path", c.caddyfilePath)
+	}
-		// Mark the cache stale so the next container change retries the load even if the container set is unchanged.
+	if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
-		c.lastFingerprint = nil
+		return fmt.Errorf("save loaded Caddyfile: %w", err)
-		// Don't write invalid config to disk.
-		return
 	}
 	c.lastFingerprint = fingerprint
-
+	c.lastCaddyCtrID = caddyCtr.ID
-	// Config loaded successfully, now write it to disk.
+	c.lastStartedAt = caddyCtr.State.StartedAt
-	if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
+	c.lastGlobal = caddyCtr.ServiceSpec.CaddyConfig()
-		c.log.Error("Failed to write Caddyfile to disk after successful load.", "err", err)
+	c.lastSavedBody = caddyfileBody(caddyfile)
-		// Config is already loaded in Caddy, so this is not critical. The next regeneration retries the disk write.
-		return
-	}
-
 	c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath)
+
+	return nil
+}
+
+// isBootstrapCaddyfile distinguishes a reduced startup file from a full file that must survive Caddy outage.
+// It also recognizes the older daemon's offline file. That file omitted every x-caddy block, including globals,
+// so treating it as a protected full file would preserve the configuration-loss bug:
+// https://github.com/psviderski/uncloud/issues/412
+func isBootstrapCaddyfile(caddyfile string) bool {
+	return strings.Contains(caddyfile, bootstrapMarker) || strings.Contains(caddyfile, legacyBootstrapMarker)
 }
 
 // fingerprintContainers returns a fingerprint of containers that the Caddyfile generator depends on.
 func fingerprintContainers(containers []store.ContainerRecord) []containerFingerprint {
 	fingerprints := make([]containerFingerprint, len(containers))
 	for i, cr := range containers {
-		// Ignore ports parsing error as not much we can do about it. The generator just logs them and continues.
+		// Ignore ports parsing error as not much we can do about it. The generator just logs them and skips
+		// the container.
 		ports, _ := cr.Container.ServicePorts()
 		fingerprints[i] = containerFingerprint{
 			ID:          cr.Container.ID,
@@ -220,21 +288,71 @@ func fingerprintContainers(containers []store.ContainerRecord) []containerFinger
 	return fingerprints
 }
 
-// writeCaddyfileIfChanged writes the Caddyfile content to disk with proper permissions only if its body differs
+// selectLocalCaddyContainer chooses which local Caddy container supplies global Caddy config. A running container
-// from the last successfully written content. The first line of the Caddyfile carries a generation timestamp that
+// takes precedence over a newer stopped replacement, even if the running container is unhealthy. If two revisions
-// changes on every regeneration, so it's excluded from the comparison to avoid redundant writes.
+// run at once, the shared admin socket does not identify its owner, so this choice remains best effort.
+func selectLocalCaddyContainer(records []store.ContainerRecord, machineID string) *api.ServiceContainer {
+	var selected *api.ServiceContainer
+	for _, cr := range records {
+		ctr := cr.Container
+		if cr.MachineID != machineID || ctr.ServiceName() != CaddyServiceName || ctr.IsHook() {
+			continue
+		}
+		if selected == nil {
+			selected = &ctr
+			continue
+		}
+		if ctr.State.Running != selected.State.Running {
+			if ctr.State.Running {
+				selected = &ctr
+			}
+			continue
+		}
+		if ctr.CreatedTime().Compare(selected.CreatedTime()) > 0 {
+			selected = &ctr
+		}
+	}
+
+	return selected
+}
+
+// writeCaddyfileIfChanged atomically replaces the saved Caddyfile only if its body differs from the new content.
 func (c *Controller) writeCaddyfileIfChanged(caddyfile string) error {
-	if caddyfileBody(caddyfile) == caddyfileBody(c.lastCaddyfile) {
+	saved, err := os.ReadFile(c.caddyfilePath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read Caddyfile '%s': %w", c.caddyfilePath, err)
+	}
+	if err == nil && caddyfileBody(caddyfile) == caddyfileBody(string(saved)) {
 		return nil
 	}
 
-	if err := os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil {
+	dir := filepath.Dir(c.caddyfilePath)
-		return fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err)
+	tmp, err := os.CreateTemp(dir, ".Caddyfile-*")
+	if err != nil {
+		return fmt.Errorf("create temporary Caddyfile: %w", err)
 	}
-	if err := fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil {
+	defer os.Remove(tmp.Name())
-		return fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err)
+	defer tmp.Close()
+
+	if err = tmp.Chmod(0o640); err != nil {
+		return fmt.Errorf("set temporary Caddyfile permissions: %w", err)
+	}
+	if err = fs.Chown(tmp.Name(), "", c.fileGroup); err != nil {
+		return fmt.Errorf("change owner of temporary Caddyfile: %w", err)
+	}
+
+	if _, err = tmp.WriteString(caddyfile); err != nil {
+		return fmt.Errorf("write temporary Caddyfile: %w", err)
+	}
+	if err = tmp.Sync(); err != nil {
+		return fmt.Errorf("sync temporary Caddyfile: %w", err)
+	}
+	if err = tmp.Close(); err != nil {
+		return fmt.Errorf("close temporary Caddyfile: %w", err)
+	}
+	if err = os.Rename(tmp.Name(), c.caddyfilePath); err != nil {
+		return fmt.Errorf("replace Caddyfile '%s': %w", c.caddyfilePath, err)
 	}
-	c.lastCaddyfile = caddyfile
 
 	return nil
 }
@@ -247,30 +365,3 @@ func caddyfileBody(caddyfile string) string {
 	}
 	return caddyfile
 }
-
-func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
-	serviceContainers := make([]api.ServiceContainer, len(containers))
-	for i, cr := range containers {
-		serviceContainers[i] = cr.Container
-	}
-
-	config, err := GenerateJSONConfig(serviceContainers, c.machineID)
-	if err != nil {
-		return err
-	}
-
-	configBytes, err := json.MarshalIndent(config, "", "    ")
-	if err != nil {
-		return fmt.Errorf("marshal Caddy configuration: %w", err)
-	}
-	configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json")
-
-	if err = os.WriteFile(configPath, configBytes, 0o640); err != nil {
-		return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err)
-	}
-	if err = fs.Chown(configPath, "", CaddyGroup); err != nil {
-		return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err)
-	}
-
-	return nil
-}

internal/machine/caddyconfig/controller_test.go

@@ -1,14 +1,488 @@
 package caddyconfig
 
 import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
 	"net/netip"
+	"os"
+	"os/user"
+	"path/filepath"
 	"reflect"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"github.com/docker/docker/api/types/container"
+	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/pkg/api"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func testController(t *testing.T, socketPath string) *Controller {
+	t.Helper()
+	group, err := user.LookupGroupId(strconv.Itoa(os.Getegid()))
+	require.NoError(t, err)
+	path := filepath.Join(t.TempDir(), "Caddyfile")
+	client := NewCaddyAdminClient(socketPath)
+	return &Controller{
+		machineID:     "test-machine-id",
+		caddyfilePath: path,
+		generator:     NewCaddyfileGenerator("test-machine-id", "test-machine", client, nil),
+		client:        client,
+		log:           slog.Default(),
+		fileGroup:     group.Name,
+	}
+}
+
+type testCaddyAdmin struct {
+	mu                    sync.Mutex
+	adaptRequests         []string
+	loadCount             int
+	rejectAdaptContaining string
+	rejectLoad            bool
+}
+
+func (a *testCaddyAdmin) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/adapt":
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		a.mu.Lock()
+		a.adaptRequests = append(a.adaptRequests, string(body))
+		reject := a.rejectAdaptContaining != "" && strings.Contains(string(body), a.rejectAdaptContaining)
+		a.mu.Unlock()
+		if reject {
+			http.Error(w, "invalid Caddyfile", http.StatusBadRequest)
+			return
+		}
+		_, _ = io.WriteString(w, `{"result":{}}`)
+	case "/load":
+		a.mu.Lock()
+		a.loadCount++
+		reject := a.rejectLoad
+		a.mu.Unlock()
+		if reject {
+			http.Error(w, "load rejected", http.StatusBadRequest)
+		}
+	default:
+		http.NotFound(w, r)
+	}
+}
+
+func (a *testCaddyAdmin) snapshot() ([]string, int) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return append([]string(nil), a.adaptRequests...), a.loadCount
+}
+
+func (a *testCaddyAdmin) setRejectLoad(reject bool) {
+	a.mu.Lock()
+	a.rejectLoad = reject
+	a.mu.Unlock()
+}
+
+func TestController_GenerateAndLoadCaddyfile(t *testing.T) {
+	t.Run("keeps the saved file when there is no local Caddy container", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		saved := "# previous full Caddyfile\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ remote }",
+			"other-machine", time.Now())
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{remote}))
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+	})
+
+	t.Run("does not publish a bootstrap with an invalid global template", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{{upstreams",
+			"test-machine-id", time.Now())
+
+		require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(),
+			[]store.ContainerRecord{caddyCtr}), "render template")
+		_, err := os.Stat(controller.caddyfilePath)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+
+	t.Run("bootstraps unhealthy Caddy with its globals and healthy routes", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"}
+		newerStopped := newContainerRecordWithCaddyConfig("caddy", "10.210.0.6", "{\n\tstorage replacement\n}",
+			"test-machine-id", time.Now().Add(time.Minute))
+		newerStopped.Container.State.Running = false
+		app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"},
+			"test-machine-id")
+		app.Container.ServiceSpec.Caddy = &api.CaddySpec{Config: "custom.example.com { respond app }"}
+		unhealthy := newContainerRecordWithPorts("unhealthy", "10.210.0.4",
+			[]string{"unhealthy.example.com:8080/http"}, "test-machine-id")
+		unhealthy.Container.State.Health = &container.Health{Status: "unhealthy"}
+		hook := newContainerRecordWithPorts("hook", "10.210.0.5",
+			[]string{"hook.example.com:8080/http"}, "test-machine-id")
+		hook.Container.Config.Labels[api.LabelHook] = "pre-deploy"
+
+		err := controller.generateAndLoadCaddyfile(context.Background(),
+			[]store.ContainerRecord{newerStopped, caddyCtr, app, unhealthy, hook})
+		require.ErrorContains(t, err, "admin socket unavailable")
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
+		assert.Contains(t, string(saved), "storage uncloud")
+		assert.NotContains(t, string(saved), "storage replacement")
+		assert.Contains(t, string(saved), "app.example.com")
+		assert.NotContains(t, string(saved), "custom.example.com")
+		assert.NotContains(t, string(saved), "unhealthy.example.com")
+		assert.NotContains(t, string(saved), "hook.example.com")
+	})
+
+	t.Run("replaces a legacy offline file with a bootstrap containing globals", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		legacy := "# generated by older daemon\nhttp:// { respond ok }\n" + legacyBootstrapMarker + "\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(legacy), 0o640))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+
+		err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr})
+		require.ErrorContains(t, err, "admin socket unavailable")
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), "storage uncloud")
+		assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
+		assert.NotContains(t, string(saved), legacyBootstrapMarker)
+	})
+
+	t.Run("saves a bootstrap for a stopped Caddy container with no file", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		caddyCtr.Container.State.Running = false
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr}))
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
+		assert.Contains(t, string(saved), "storage uncloud")
+	})
+
+	for _, tc := range []struct {
+		name    string
+		running bool
+	}{
+		{name: "preserves a full file when the admin socket is unavailable", running: true},
+		{name: "preserves a stopped container's full file", running: false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+			saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n"
+			require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+			caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage new\n}",
+				"test-machine-id", time.Now())
+			caddyCtr.Container.State.Running = tc.running
+			app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"new.example.com:8080/http"},
+				"test-machine-id")
+
+			err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app})
+			if tc.running {
+				require.ErrorContains(t, err, "admin socket unavailable")
+			} else {
+				require.NoError(t, err)
+			}
+			got, err := os.ReadFile(controller.caddyfilePath)
+			require.NoError(t, err)
+			assert.Equal(t, saved, string(got))
+		})
+	}
+
+	t.Run("loads a full file and reloads after Caddy restarts", func(t *testing.T) {
+		admin := &testCaddyAdmin{}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"}
+		app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "app.example.com { respond app }",
+			"test-machine-id", time.Now())
+		records := []store.ContainerRecord{caddyCtr, app}
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), "storage uncloud")
+		assert.Contains(t, string(saved), "app.example.com")
+		assert.NotContains(t, string(saved), bootstrapMarker)
+		adapted, loads := admin.snapshot()
+		require.NotEmpty(t, adapted)
+		assert.Contains(t, adapted[0], "storage uncloud")
+		assert.Equal(t, 1, loads)
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		_, loads = admin.snapshot()
+		assert.Equal(t, 1, loads, "unchanged inputs should not reload Caddy")
+
+		records[0].Container.State.StartedAt = time.Now().UTC().Format(time.RFC3339Nano)
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		_, loads = admin.snapshot()
+		assert.Equal(t, 2, loads, "a restarted Caddy process needs the full configuration")
+	})
+
+	t.Run("reloads when an application route changes", func(t *testing.T) {
+		admin := &testCaddyAdmin{}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"},
+			"test-machine-id")
+		records := []store.ContainerRecord{caddyCtr, app}
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		_, loads := admin.snapshot()
+		assert.Equal(t, 1, loads)
+
+		records[1] = newContainerRecordWithPorts("web", "10.210.0.4",
+			[]string{"app.example.com:8080/http"}, "test-machine-id")
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), "10.210.0.4:8080")
+		assert.NotContains(t, string(saved), "10.210.0.3:8080")
+		_, loads = admin.snapshot()
+		assert.Equal(t, 2, loads)
+	})
+
+	t.Run("rejects invalid globals without loading or replacing the saved file", func(t *testing.T) {
+		admin := &testCaddyAdmin{rejectAdaptContaining: "storage rejected"}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage rejected\n}",
+			"test-machine-id", time.Now())
+
+		err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr})
+		require.ErrorContains(t, err, "validate user-defined global Caddy config")
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		_, loads := admin.snapshot()
+		assert.Zero(t, loads)
+	})
+
+	t.Run("skips invalid application custom config", func(t *testing.T) {
+		admin := &testCaddyAdmin{rejectAdaptContaining: "invalid.example.com"}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		invalid := newContainerRecordWithCaddyConfig("invalid", "10.210.0.3",
+			"invalid.example.com { respond bad }", "test-machine-id", time.Now())
+		valid := newContainerRecordWithCaddyConfig("valid", "10.210.0.4",
+			"valid.example.com { respond good }", "test-machine-id", time.Now())
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(),
+			[]store.ContainerRecord{caddyCtr, invalid, valid}))
+		saved, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(saved), "storage uncloud")
+		assert.Contains(t, string(saved), "valid.example.com")
+		assert.NotContains(t, string(saved), "invalid.example.com")
+		assert.Contains(t, string(saved), "Skipped invalid user-defined configs")
+		_, loads := admin.snapshot()
+		assert.Equal(t, 1, loads)
+	})
+
+	t.Run("keeps the saved file when application validation cannot complete", func(t *testing.T) {
+		var loads atomic.Int64
+		socket := testAdminServer(t, func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/adapt":
+				http.Error(w, "adapter unavailable", http.StatusInternalServerError)
+			case "/load":
+				loads.Add(1)
+			default:
+				http.NotFound(w, r)
+			}
+		})
+		controller := testController(t, socket)
+		saved := "old.example.com { respond old }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "",
+			"test-machine-id", time.Now())
+		app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "new.example.com { respond new }",
+			"test-machine-id", time.Now())
+
+		err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app})
+		require.ErrorContains(t, err, "validate Caddy config for service")
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		assert.Zero(t, loads.Load())
+	})
+
+	t.Run("retries a rejected load without changing inputs", func(t *testing.T) {
+		admin := &testCaddyAdmin{rejectLoad: true}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		records := []store.ContainerRecord{caddyCtr}
+
+		require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "load Caddyfile")
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		admin.setRejectLoad(false)
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		got, err = os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(got), "storage uncloud")
+		_, loads := admin.snapshot()
+		assert.Equal(t, 2, loads)
+	})
+
+	t.Run("retries saving after a successful load", func(t *testing.T) {
+		admin := &testCaddyAdmin{}
+		controller := testController(t, testAdminServer(t, admin.ServeHTTP))
+		group := controller.fileGroup
+		controller.fileGroup = "uncloud-test-group-that-does-not-exist"
+		saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		records := []store.ContainerRecord{caddyCtr}
+
+		require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "save loaded Caddyfile")
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		controller.fileGroup = group
+
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		got, err = os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(got), "storage uncloud")
+		_, loads := admin.snapshot()
+		assert.Equal(t, 2, loads)
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		_, loads = admin.snapshot()
+		assert.Equal(t, 2, loads, "a loaded and saved revision should not reload")
+	})
+
+	t.Run("loads a saved bootstrap when the admin socket becomes available", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
+			"test-machine-id", time.Now())
+		records := []store.ContainerRecord{caddyCtr}
+		require.Error(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		bootstrap, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Contains(t, string(bootstrap), bootstrapMarker)
+
+		admin := &testCaddyAdmin{}
+		socket := testAdminServer(t, admin.ServeHTTP)
+		controller.client = NewCaddyAdminClient(socket)
+		controller.generator = NewCaddyfileGenerator("test-machine-id", "test-machine", controller.client, nil)
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		full, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.NotContains(t, string(full), bootstrapMarker)
+		_, loads := admin.snapshot()
+		assert.Equal(t, 1, loads)
+
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, bootstrap, 0o640))
+		require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
+		_, loads = admin.snapshot()
+		assert.Equal(t, 2, loads, "a saved bootstrap is not an applied full configuration")
+	})
+}
+
+func TestController_WriteCaddyfileIfChanged(t *testing.T) {
+	t.Run("publishes a complete file with the configured group", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		caddyfile := "# Generated now\n{\n\tstorage uncloud\n}\n"
+
+		require.NoError(t, controller.writeCaddyfileIfChanged(caddyfile))
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, caddyfile, string(got))
+		info, err := os.Stat(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, os.FileMode(0o640), info.Mode().Perm())
+	})
+
+	t.Run("does not replace a file when only the timestamp changes", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		saved := "# Generated yesterday\nsite.example.com { respond ok }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+		before, err := os.Stat(controller.caddyfilePath)
+		require.NoError(t, err)
+
+		require.NoError(t, controller.writeCaddyfileIfChanged(
+			"# Generated today\nsite.example.com { respond ok }\n"))
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		after, err := os.Stat(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.True(t, os.SameFile(before, after), "an unchanged body should not replace the file")
+	})
+
+	t.Run("preserves the previous file when publication fails", func(t *testing.T) {
+		controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
+		controller.fileGroup = "uncloud-test-group-that-does-not-exist"
+		saved := "# previous\nsite.example.com { respond old }\n"
+		require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
+
+		err := controller.writeCaddyfileIfChanged("# next\nsite.example.com { respond new }\n")
+		require.ErrorContains(t, err, "change owner of temporary Caddyfile")
+		got, err := os.ReadFile(controller.caddyfilePath)
+		require.NoError(t, err)
+		assert.Equal(t, saved, string(got))
+		tmpFiles, err := filepath.Glob(filepath.Join(filepath.Dir(controller.caddyfilePath), ".Caddyfile-*"))
+		require.NoError(t, err)
+		assert.Empty(t, tmpFiles)
+	})
+}
+
+func TestSelectLocalCaddyContainer(t *testing.T) {
+	t.Run("prefers a running unhealthy container over a newer stopped one", func(t *testing.T) {
+		older := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ older }",
+			"test-machine-id", time.Now().Add(-time.Hour))
+		older.Container.State.Health = &container.Health{Status: "unhealthy"}
+		newer := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ newer }",
+			"test-machine-id", time.Now())
+		newer.Container.State.Running = false
+
+		selected := selectLocalCaddyContainer([]store.ContainerRecord{newer, older}, "test-machine-id")
+		require.NotNil(t, selected)
+		assert.Equal(t, older.Container.ID, selected.ID)
+	})
+
+	t.Run("ignores remote and hook containers", func(t *testing.T) {
+		local := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ local }",
+			"test-machine-id", time.Now().Add(-time.Hour))
+		remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ remote }",
+			"other-machine", time.Now())
+		hook := newContainerRecordWithCaddyConfig("caddy", "10.210.0.4", "{ hook }",
+			"test-machine-id", time.Now())
+		hook.Container.Config.Labels[api.LabelHook] = "pre-deploy"
+
+		selected := selectLocalCaddyContainer([]store.ContainerRecord{remote, hook, local}, "test-machine-id")
+		require.NotNil(t, selected)
+		assert.Equal(t, local.Container.ID, selected.ID)
+	})
+}
+
 // TestContainerFingerprint_EqualCoversAllFields is a guard: when a field is added to containerFingerprint,
 // Equal must also compare it. A mutation of any single field should flip equality to false. If this test fails
 // after adding a field, update Equal to include it.

internal/machine/caddyconfig/jsonconfig.go

@@ -1,141 +0,0 @@
-package caddyconfig
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"maps"
-	"net/http"
-	"slices"
-	"strconv"
-	"time"
-
-	"github.com/caddyserver/caddy/v2"
-	"github.com/caddyserver/caddy/v2/caddyconfig"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
-	"github.com/psviderski/uncloud/pkg/api"
-)
-
-func GenerateJSONConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) {
-	httpHostUpstreams, httpsHostUpstreams := httpUpstreamsFromPorts(containers)
-
-	var warnings []caddyconfig.Warning
-	servers := make(map[string]*caddyhttp.Server)
-	servers["http"] = &caddyhttp.Server{
-		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
-		// All http requests to this server are logged to the default logger.
-		Logs: &caddyhttp.ServerLogConfig{},
-		Routes: append(
-			hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
-			// Add a route to respond with a static verification response at the /.uncloud-verify path.
-			verificationRoute(verifyResponse, &warnings),
-		),
-	}
-	servers["https"] = &caddyhttp.Server{
-		Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
-		// All https requests to this server are logged to the default logger.
-		Logs:   &caddyhttp.ServerLogConfig{},
-		Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
-	}
-
-	httpApp := caddyhttp.App{
-		Servers: servers,
-	}
-	config := &caddy.Config{
-		AppsRaw: caddy.ModuleMap{
-			"http": caddyconfig.JSON(httpApp, &warnings),
-		},
-	}
-
-	var err error
-	if len(warnings) > 0 {
-		// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
-		for _, w := range warnings {
-			err = errors.Join(err, errors.New(w.Message))
-		}
-		return nil, fmt.Errorf("marshal Caddy configuration: %w", err)
-	}
-
-	return config, nil
-}
-
-// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
-func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
-	// Sort hostnames for deterministic output.
-	hostnames := slices.Collect(maps.Keys(hostUpstreams))
-	slices.Sort(hostnames)
-
-	routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
-	for _, hostname := range hostnames {
-		upstreams := hostUpstreams[hostname]
-		upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
-		for i, upstream := range upstreams {
-			upstreamPool[i] = &reverseproxy.Upstream{
-				Dial: upstream,
-			}
-		}
-		handler := &reverseproxy.Handler{
-			HealthChecks: &reverseproxy.HealthChecks{
-				// Enable passive health checks to automatically detect unhealthy upstreams.
-				Passive: &reverseproxy.PassiveHealthChecks{
-					FailDuration: caddy.Duration(30 * time.Second),
-				},
-			},
-			LoadBalancing: &reverseproxy.LoadBalancing{
-				// Retry failed requests to skip over temporarily unavailable upstreams.
-				Retries: 3,
-			},
-			Upstreams: upstreamPool,
-		}
-
-		routes = append(routes, caddyhttp.Route{
-			MatcherSetsRaw: caddyhttp.RawMatcherSets{
-				{
-					"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
-				},
-			},
-			HandlersRaw: []json.RawMessage{
-				caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
-			},
-		})
-	}
-	return routes
-}
-
-// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
-func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
-	// Return the following route:
-	// {
-	//   "match": [
-	//     {
-	//       "path": [
-	//         "/.uncloud-verify"
-	//       ]
-	//     }
-	//   ],
-	//   "handle": [
-	//     {
-	//       "handler": "static_response",
-	//       "body": "<response>",
-	//       "status_code": 200
-	//     }
-	//   ]
-	// }
-
-	staticResponse := caddyhttp.StaticResponse{
-		StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
-		Body:       response,
-	}
-
-	return caddyhttp.Route{
-		MatcherSetsRaw: caddyhttp.RawMatcherSets{
-			{
-				"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
-			},
-		},
-		HandlersRaw: []json.RawMessage{
-			caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
-		},
-	}
-}

internal/machine/caddyconfig/jsonconfig_test.go

@@ -1,378 +0,0 @@
-package caddyconfig
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
-	"github.com/psviderski/uncloud/internal/machine/docker"
-	"github.com/psviderski/uncloud/pkg/api"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestGenerateJSONConfig(t *testing.T) {
-	configWithoutServices := `{
-		"servers": {
-			"http": {
-				"listen": [":80"],
-				"routes": [{
-					"match": [{"path": ["/.uncloud-verify"]}],
-					"handle": [{
-						"body": "verification-response-body",
-						"handler": "static_response",
-						"status_code": 200
-					}]
-				}],
-				"logs": {}
-			},
-			"https": {
-				"listen": [":443"],
-				"logs": {}
-			}
-		}
-	}`
-
-	tests := []struct {
-		name       string
-		containers []api.ServiceContainer
-		want       string
-		wantErr    bool
-	}{
-		{
-			name:       "empty containers",
-			containers: []api.ServiceContainer{},
-			want:       configWithoutServices,
-			wantErr:    false,
-		},
-
-		{
-			name: "HTTP container",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
-			},
-			want: `{
-                "servers": {
-                    "http": {
-                        "listen": [":80"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["app.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [{"dial": "10.210.0.2:8080"}]
-                                }]
-                            },
-                            {
-                                "match": [{"path": ["/.uncloud-verify"]}],
-                                "handle": [{
-                                    "body": "verification-response-body",
-                                    "handler": "static_response",
-                                    "status_code": 200
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    },
-                    "https": {
-                        "listen": [":443"],
-						"logs": {}
-                    }
-                }
-            }`,
-			wantErr: false,
-		},
-		{
-			name: "load balancing multiple containers",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "app.example.com:8080/http"),
-				newContainer("10.210.0.3", "app.example.com:8080/http"),
-			},
-			want: `{
-                "servers": {
-                    "http": {
-                        "listen": [":80"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["app.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [
-                                        {"dial": "10.210.0.2:8080"},
-                                        {"dial": "10.210.0.3:8080"}
-                                    ]
-                                }]
-                            },
-                            {
-                                "match": [{"path": ["/.uncloud-verify"]}],
-                                "handle": [{
-                                    "body": "verification-response-body",
-                                    "handler": "static_response",
-                                    "status_code": 200
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    },
-                    "https": {
-                        "listen": [":443"],
-						"logs": {}
-                    }
-                }
-            }`,
-			wantErr: false,
-		},
-		{
-			name: "HTTPS container",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "secure.example.com:8000/https"),
-			},
-			want: `{
-                "servers": {
-                    "http": {
-                        "listen": [":80"],
-                        "routes": [
-                            {
-                                "match": [{"path": ["/.uncloud-verify"]}],
-                                "handle": [{
-                                    "body": "verification-response-body",
-                                    "handler": "static_response",
-                                    "status_code": 200
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    },
-                    "https": {
-                        "listen": [":443"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["secure.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [{"dial": "10.210.0.2:8000"}]
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    }
-                }
-            }`,
-			wantErr: false,
-		},
-		{
-			name: "mixed HTTP and HTTPS",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2",
-					"app.example.com:8080/http",
-					"web.example.com:8000/http"),
-				newContainer("10.210.0.3",
-					"app.example.com:8080/http",
-					"secure.example.com:8888/https"),
-				newContainer("10.210.0.4",
-					"web.example.com:8000/http",
-					"secure.example.com:8888/https"),
-				newContainer("10.210.0.5",
-					"app.example.com:8080/http",
-					"web.example.com:8000/http",
-					"secure.example.com:8888/https"),
-			},
-			want: `{
-                "servers": {
-                    "http": {
-                        "listen": [":80"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["app.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [
-										{"dial": "10.210.0.2:8080"},
-										{"dial": "10.210.0.3:8080"},
-										{"dial": "10.210.0.5:8080"}
-									]
-                                }]
-                            },
-							{
-                                "match": [{"host": ["web.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [
-										{"dial": "10.210.0.2:8000"},
-										{"dial": "10.210.0.4:8000"},
-										{"dial": "10.210.0.5:8000"}
-									]
-                                }]
-                            },
-                            {
-                                "match": [{"path": ["/.uncloud-verify"]}],
-                                "handle": [{
-                                    "body": "verification-response-body",
-                                    "handler": "static_response",
-                                    "status_code": 200
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    },
-                    "https": {
-                        "listen": [":443"],
-                        "routes": [
-                            {
-                                "match": [{"host": ["secure.example.com"]}],
-                                "handle": [{
-                                    "handler": "reverse_proxy",
-									"health_checks": {
-                                        "passive": {
-                                            "fail_duration": 30000000000
-                                        }
-                                    },
-                                    "load_balancing": {
-                                        "retries": 3
-                                    },
-                                    "upstreams": [
-										{"dial": "10.210.0.3:8888"},
-										{"dial": "10.210.0.4:8888"},
-										{"dial": "10.210.0.5:8888"}
-									]
-                                }]
-                            }
-                        ],
-						"logs": {}
-                    }
-                }
-            }`,
-			wantErr: false,
-		},
-		{
-			name: "container without uncloud network ignored",
-			containers: []api.ServiceContainer{
-				newContainerWithoutNetwork("ignored.example.com:8080/http"),
-			},
-			want:    configWithoutServices,
-			wantErr: false,
-		},
-		{
-			name: "container with invalid port ignored",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "invalid-port"),
-			},
-			want:    configWithoutServices,
-			wantErr: false,
-		},
-		{
-			name: "containers with unsupported protocols and host mode ignored",
-			containers: []api.ServiceContainer{
-				newContainer("10.210.0.2", "5000/tcp"),
-				newContainer("10.210.0.3", "5000/udp"),
-				newContainer("10.210.0.4", "80:8080/tcp@host"),
-			},
-			want:    configWithoutServices,
-			wantErr: false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			config, err := GenerateJSONConfig(tt.containers, "verification-response-body")
-
-			if tt.wantErr {
-				assert.Error(t, err)
-				return
-			}
-			require.NoError(t, err)
-
-			require.Len(t, config.AppsRaw, 1, "Expected one http app")
-			require.Contains(t, config.AppsRaw, "http", "Expected http app")
-
-			assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match")
-		})
-	}
-}
-
-func newContainer(ip string, ports ...string) api.ServiceContainer {
-	portsLabel := strings.Join(ports, ",")
-	return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
-		ContainerJSONBase: &container.ContainerJSONBase{
-			State: &container.State{
-				Running: true,
-			},
-		},
-		NetworkSettings: &container.NetworkSettings{
-			Networks: map[string]*network.EndpointSettings{
-				docker.NetworkName: {
-					IPAddress: ip,
-				},
-			},
-		},
-		Config: &container.Config{
-			Labels: map[string]string{
-				api.LabelServicePorts: portsLabel,
-			},
-		},
-	}}}
-}
-
-func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
-	portsLabel := strings.Join(ports, ",")
-	return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
-		ContainerJSONBase: &container.ContainerJSONBase{
-			State: &container.State{
-				Running: true,
-			},
-		},
-		NetworkSettings: &container.NetworkSettings{
-			Networks: map[string]*network.EndpointSettings{
-				"other-network": {
-					IPAddress: "172.17.0.2",
-				},
-			},
-		},
-		Config: &container.Config{
-			Labels: map[string]string{
-				api.LabelServicePorts: portsLabel,
-			},
-		},
-	}}}
-}

internal/machine/caddyconfig/server.go

@@ -2,6 +2,7 @@ package caddyconfig
 
 import (
 	"context"
+	"errors"
 	"os"
 
 	"google.golang.org/grpc/codes"
@@ -22,18 +23,26 @@ func NewServer(service *Service) *Server {
 	return &Server{service: service}
 }
 
-// GetConfig retrieves the current Caddy configuration from the machine.
+// GetConfig retrieves the saved Caddy configuration and the latest reconciliation error from the machine.
 func (s *Server) GetConfig(ctx context.Context, _ *emptypb.Empty) (*pb.GetCaddyConfigResponse, error) {
 	caddyfile, modifiedAt, err := s.service.Caddyfile()
 	if err != nil {
-		if os.IsNotExist(err) {
+		if errors.Is(err, os.ErrNotExist) {
+			if lastErr := s.service.LastReconciliationError(); lastErr != nil {
+				return nil, status.Errorf(codes.NotFound, "%v; last Caddy config load failed: %v", err, lastErr)
+			}
 			return nil, status.Error(codes.NotFound, err.Error())
 		}
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	recErr := ""
+	if lastErr := s.service.LastReconciliationError(); lastErr != nil {
+		recErr = lastErr.Error()
+	}
 	return &pb.GetCaddyConfigResponse{
 		Caddyfile:               caddyfile,
 		ModifiedAt:              timestamppb.New(modifiedAt),
+		LastReconciliationError: recErr,
 	}, nil
 }

internal/machine/caddyconfig/server_test.go

@@ -0,0 +1,62 @@
+package caddyconfig
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+func TestServer_GetConfig(t *testing.T) {
+	t.Run("returns saved Caddyfile and reconciliation error", func(t *testing.T) {
+		dir := t.TempDir()
+		caddyfile := "example.com { respond ok }\n"
+		path := filepath.Join(dir, "Caddyfile")
+		require.NoError(t, os.WriteFile(path, []byte(caddyfile), 0o640))
+		info, err := os.Stat(path)
+		require.NoError(t, err)
+
+		service := NewService(dir)
+		service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud"))
+		config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
+
+		require.NoError(t, err)
+		assert.Equal(t, caddyfile, config.GetCaddyfile())
+		assert.True(t, info.ModTime().Equal(config.GetModifiedAt().AsTime()))
+		assert.Equal(t, "module not registered: caddy.storage.uncloud", config.GetLastReconciliationError())
+
+		service.setReconciliationResult(nil)
+		config, err = NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
+		require.NoError(t, err)
+		assert.Empty(t, config.GetLastReconciliationError())
+	})
+
+	t.Run("returns NotFound with reconciliation error when Caddyfile is absent", func(t *testing.T) {
+		service := NewService(t.TempDir())
+		service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud"))
+
+		config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
+
+		assert.Nil(t, config)
+		assert.Equal(t, codes.NotFound, status.Code(err))
+		assert.ErrorContains(t, err, "Caddyfile")
+		assert.ErrorContains(t, err, "last Caddy config load failed: module not registered: caddy.storage.uncloud")
+	})
+
+	t.Run("returns NotFound without reconciliation error when Caddyfile is absent", func(t *testing.T) {
+		service := NewService(t.TempDir())
+
+		config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
+
+		assert.Nil(t, config)
+		assert.Equal(t, codes.NotFound, status.Code(err))
+		assert.NotContains(t, err.Error(), "last Caddy config load failed")
+	})
+}

internal/machine/caddyconfig/service.go

@@ -4,12 +4,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 )
 
 // Service provides methods to interact with the Caddy configuration on the machine.
 type Service struct {
 	configDir               string
+	mu                      sync.RWMutex
+	lastReconciliationError error
 }
 
 // NewService creates a new Service instance with the specified Caddy configuration directory.
@@ -17,7 +20,8 @@ func NewService(configDir string) *Service {
 	return &Service{configDir: configDir}
 }
 
-// Caddyfile retrieves the current Caddy configuration (Caddyfile) from the machine's config directory.
+// Caddyfile retrieves the saved Caddyfile from the machine's config directory. The saved file may differ from the
+// running configuration if Caddy accepted a load but the subsequent write failed.
 func (s *Service) Caddyfile() (string, time.Time, error) {
 	path := filepath.Join(s.configDir, "Caddyfile")
 	content, err := os.ReadFile(path)
@@ -33,3 +37,17 @@ func (s *Service) Caddyfile() (string, time.Time, error) {
 
 	return string(content), fileInfo.ModTime(), nil
 }
+
+// LastReconciliationError reports the most recent unsuccessful controller attempt, if any. A successful attempt
+// clears it. An empty value does not prove that Caddy has loaded the saved Caddyfile.
+func (s *Service) LastReconciliationError() error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.lastReconciliationError
+}
+
+func (s *Service) setReconciliationResult(err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.lastReconciliationError = err
+}

internal/machine/caddyconfig/service_test.go

@@ -0,0 +1,42 @@
+package caddyconfig
+
+import (
+	"errors"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestService_LastReconciliationError(t *testing.T) {
+	service := NewService(t.TempDir())
+	assert.NoError(t, service.LastReconciliationError())
+
+	service.setReconciliationResult(errors.New("global Caddy config rejected"))
+	assert.EqualError(t, service.LastReconciliationError(), "global Caddy config rejected")
+
+	service.setReconciliationResult(nil)
+	assert.NoError(t, service.LastReconciliationError())
+}
+
+func TestService_LastReconciliationErrorConcurrent(t *testing.T) {
+	service := NewService(t.TempDir())
+	var wg sync.WaitGroup
+	for range 10 {
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			for range 100 {
+				service.setReconciliationResult(errors.New("retry"))
+				service.setReconciliationResult(nil)
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			for range 100 {
+				_ = service.LastReconciliationError()
+			}
+		}()
+	}
+	wg.Wait()
+}

internal/machine/machine.go

@@ -198,6 +198,8 @@ type Machine struct {
 	dockerServer  *machinedocker.Server
 	// machineAPIServer handles API requests directly on this machine.
 	machineAPIServer *grpc.Server
+	// caddyService provides methods to interact with the Caddy configuration on the machine.
+	caddyService *caddyconfig.Service
 
 	// proxyDirector routes API requests to local or remote machines.
 	proxyDirector *apiproxy.Director
@@ -302,6 +304,7 @@ func NewMachine(config *Config) (*Machine, error) {
 		dockerService:    dockerService,
 		clusterAPIServer: clusterAPIServer,
 		proxyDirector:    proxyDirector,
+		caddyService:     caddyconfig.NewService(config.CaddyConfigDir),
 	}
 
 	// Machine IP will only be available after the machine is initialised as a cluster member so wrap it in a function.
@@ -316,7 +319,7 @@ func NewMachine(config *Config) (*Machine, error) {
 		NetworkReady:        m.IsNetworkReady,
 		WaitForNetworkReady: m.WaitForNetworkReady,
 	})
-	caddyServer := caddyconfig.NewServer(caddyconfig.NewService(config.CaddyConfigDir))
+	caddyServer := caddyconfig.NewServer(m.caddyService)
 
 	caddyStore, err := corroStore.Keyspace(caddystorage.Namespace)
 	if err != nil {
@@ -535,7 +538,7 @@ func (m *Machine) Run(ctx context.Context) error {
 			// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
 			caddyconfigCtrl, err := caddyconfig.NewController(
 				m.state.ID,
-				m.config.CaddyConfigDir,
+				m.caddyService,
 				DefaultCaddyAdminSockPath,
 				m.store,
 			)

test/e2e/machine_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/psviderski/uncloud/api/pb"
 	"github.com/psviderski/uncloud/internal/ucind"
 	"github.com/psviderski/uncloud/pkg/api"
+	"github.com/psviderski/uncloud/pkg/client"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -456,6 +457,23 @@ func TestMachineOperations(t *testing.T) {
 	})
 
 	t.Run("remove machine clears container records from cluster store", func(t *testing.T) {
+		// The Caddy controller needs a local Caddy container to supply the global config before it can generate
+		// routes. Place it on the connected machine by ID because earlier tests rename that machine.
+		caddyDeployment, err := cli.NewCaddyDeployment("", "", api.Placement{
+			Machines: []string{c.Machines[0].ID},
+		})
+		require.NoError(t, err)
+		monitorPeriod := 5 * time.Second
+		caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &monitorPeriod
+		_, err = caddyDeployment.Run(ctx)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := cli.RemoveService(ctx, client.CaddyServiceName)
+			if err != nil && !errors.Is(err, api.ErrNotFound) {
+				assert.NoError(t, err)
+			}
+		})
+
 		// Deploy a global service with an HTTP ingress port so the auto-generated Caddyfile lists
 		// each container's IP as an upstream.
 		serviceName := "test-machine-rm-cleanup"

test/e2e/service_test.go

@@ -27,6 +27,8 @@ import (
 	"github.com/psviderski/uncloud/pkg/client/deploy"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func newServiceID() string {
@@ -42,6 +44,9 @@ func TestDeployment(t *testing.T) {
 
 	clusterName := "ucind-test.deployment"
 	ctx := context.Background()
+	// Caddy may restart once while the controller writes its bootstrap file. Use the normal deployment monitor period
+	// instead of the zero-duration override used by most e2e tests.
+	caddyMonitorPeriod := 5 * time.Second
 	c, _ := createTestCluster(t, clusterName, ucind.CreateClusterOptions{Machines: 3}, true)
 
 	cli, cErr := c.Machines[0].Connect(ctx)
@@ -313,6 +318,7 @@ func TestDeployment(t *testing.T) {
 
 		deployment, err := cli.NewCaddyDeployment("", "", api.Placement{})
 		require.NoError(t, err)
+		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
 		_, err = deployment.Run(ctx)
 		require.NoError(t, err)
@@ -345,6 +351,7 @@ func TestDeployment(t *testing.T) {
 			Machines: []string{c.Machines[0].Name},
 		})
 		require.NoError(t, err)
+		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 		image := deployment.Spec.Container.Image
 
 		_, err = deployment.Run(ctx)
@@ -361,6 +368,7 @@ func TestDeployment(t *testing.T) {
 		// Deploy to all machines without a placement constraint.
 		deployment, err = cli.NewCaddyDeployment(image, "", api.Placement{})
 		require.NoError(t, err)
+		deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
 		_, err = deployment.Run(ctx)
 		require.NoError(t, err)
@@ -391,6 +399,13 @@ func TestDeployment(t *testing.T) {
 			}
 		})
 
+		// Without a Caddy container, the controller has no global config to pair with application configs. Keep any
+		// previously saved Caddyfile unchanged rather than publishing a new one without Caddy's global settings.
+		savedBefore, savedBeforeErr := cli.Caddy.GetConfig(ctx, nil)
+		if savedBeforeErr != nil {
+			require.Equal(t, codes.NotFound, status.Code(savedBeforeErr))
+		}
+
 		// First deploy a service with custom caddy config before caddy is deployed.
 		serviceCaddyfile := `test-custom-caddy-config.example.com {
 	reverse_proxy {{upstreams}} {
@@ -416,17 +431,13 @@ func TestDeployment(t *testing.T) {
 		require.NoError(t, err)
 		assertServiceMatchesSpec(t, svc, spec)
 
-		// Check that the generated Caddyfile contains a comment that user-define configs were skipped.
+		require.Never(t, func() bool {
-		var config *pb.GetCaddyConfigResponse
+			current, currentErr := cli.Caddy.GetConfig(ctx, nil)
-		require.Eventually(t, func() bool {
+			if savedBeforeErr != nil {
-			config, err = cli.Caddy.GetConfig(ctx, nil)
+				return currentErr == nil
-			if err != nil {
-				return false
 			}
-			return strings.Contains(config.Caddyfile, "# NOTE: User-defined configs for services were skipped")
+			return currentErr == nil && current.Caddyfile != savedBefore.Caddyfile
-		}, 5*time.Second, 100*time.Millisecond)
+		}, 2*time.Second, 100*time.Millisecond)
-
-		assert.NotContains(t, config.Caddyfile, "test-custom-caddy-config.example.com {")
 
 		// Now deploy caddy with custom config.
 		caddyCaddyfile := `{
@@ -438,6 +449,7 @@ myapp.example.com {
 }`
 		caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{})
 		require.NoError(t, err)
+		caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
 
 		_, err = caddyDeployment.Run(ctx)
 		require.NoError(t, err)
@@ -447,6 +459,7 @@ myapp.example.com {
 		assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec)
 
 		// Wait for the Caddyfile to be regenerated with both custom configs.
+		var config *pb.GetCaddyConfigResponse
 		require.Eventually(t, func() bool {
 			config, err = cli.Caddy.GetConfig(ctx, nil)
 			if err != nil {