diff --git a/api/pb/caddy.pb.go b/api/pb/caddy.pb.go index cdccbc5b..bedf4e00 100644 --- a/api/pb/caddy.pb.go +++ b/api/pb/caddy.pb.go @@ -27,10 +27,12 @@ type GetCaddyConfigResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The generated Caddyfile content. + // The saved Caddyfile content. Caddyfile string `protobuf:"bytes,1,opt,name=caddyfile,proto3" json:"caddyfile,omitempty"` // Timestamp when the config was last modified. ModifiedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=modified_at,json=modifiedAt,proto3" json:"modified_at,omitempty"` + // Error from the latest unsuccessful reconciliation attempt. + LastReconciliationError string `protobuf:"bytes,3,opt,name=last_reconciliation_error,json=lastReconciliationError,proto3" json:"last_reconciliation_error,omitempty"` } func (x *GetCaddyConfigResponse) Reset() { @@ -79,6 +81,13 @@ func (x *GetCaddyConfigResponse) GetModifiedAt() *timestamppb.Timestamp { return nil } +func (x *GetCaddyConfigResponse) GetLastReconciliationError() string { + if x != nil { + return x.LastReconciliationError + } + return "" +} + var File_api_pb_caddy_proto protoreflect.FileDescriptor var file_api_pb_caddy_proto_rawDesc = []byte{ @@ -87,22 +96,26 @@ var file_api_pb_caddy_proto_rawDesc = []byte{ 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x61, - 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x12, - 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x32, 0x49, 0x0a, 0x05, - 0x43, 0x61, 0x64, 0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, - 0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xaf, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, + 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, + 0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, 0x0a, + 0x19, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x17, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x49, 0x0a, 0x05, 0x43, 0x61, 0x64, + 0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/pb/caddy.proto b/api/pb/caddy.proto index c119d0ae..704408e7 100644 --- a/api/pb/caddy.proto +++ b/api/pb/caddy.proto @@ -13,8 +13,10 @@ service Caddy { } message GetCaddyConfigResponse { - // The generated Caddyfile content. + // The saved Caddyfile content. string caddyfile = 1; // Timestamp when the config was last modified. google.protobuf.Timestamp modified_at = 2; + // Error from the latest unsuccessful reconciliation attempt. + string last_reconciliation_error = 3; } diff --git a/cmd/uc/caddy/config.go b/cmd/uc/caddy/config.go index 6fa08260..604b4fa3 100644 --- a/cmd/uc/caddy/config.go +++ b/cmd/uc/caddy/config.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/chroma/v2/quick" "github.com/psviderski/uncloud/internal/cli" "github.com/psviderski/uncloud/internal/cli/completion" + "github.com/psviderski/uncloud/internal/cli/tui" "github.com/spf13/cobra" ) @@ -66,5 +67,10 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error { } } + if config.LastReconciliationError != "" { + tui.PrintWarning(fmt.Sprintf("last Caddy config load failed: %s\nShowing the last saved Caddyfile.", + config.LastReconciliationError)) + } + return nil } diff --git a/internal/machine/caddyconfig/caddyfile.go b/internal/machine/caddyconfig/caddyfile.go index 06c471ff..8b788d35 100644 --- a/internal/machine/caddyconfig/caddyfile.go +++ b/internal/machine/caddyconfig/caddyfile.go @@ -4,6 +4,7 @@ import ( "bytes" "cmp" "context" + "errors" "fmt" "log/slog" "maps" @@ -57,14 +58,12 @@ https://{{$hostname}} { log }{{end}} ` - caddyfileUnavailabeFooter = `# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine -# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest -# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy) -# and its logs for more details (uc logs caddy). -` + bootstrapMarker = "# Uncloud bootstrap for Caddy container " + legacyBootstrapMarker = "# NOTE: User-defined configs for services were skipped because Caddy is not running" ) // CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy. +// It combines generated routes from published ports with user-defined Caddyfile snippets from service specs (x-caddy). type CaddyfileGenerator struct { // machineID is the unique identifier of the machine where the controller is running. machineID string @@ -74,11 +73,18 @@ type CaddyfileGenerator struct { log *slog.Logger } -// CaddyfileValidator is an interface for validating Caddyfile configurations. +// CaddyfileValidator checks a candidate Caddyfile without loading it. Validate returns InvalidCaddyfileError only +// when the candidate is known to be invalid. Other errors mean the check could not be completed. A successful check +// does not guarantee that the configuration will load. type CaddyfileValidator interface { Validate(ctx context.Context, caddyfile string) error } +// InvalidCaddyfileError means validation completed and the candidate Caddyfile was rejected. +type InvalidCaddyfileError struct{ Message string } + +func (e *InvalidCaddyfileError) Error() string { return e.Message } + func NewCaddyfileGenerator( machineID, machineName string, validator CaddyfileValidator, log *slog.Logger, ) *CaddyfileGenerator { @@ -93,35 +99,41 @@ func NewCaddyfileGenerator( } } -// Generate creates a Caddyfile configuration based on the provided service containers. -// The Caddyfile is generated from the service ports of the healthy containers. -// If a 'caddy' service container is running on this machine and defines a custom Caddy config (x-caddy) in its service -// spec, it will be validated and prepended to the generated Caddyfile. Custom Caddy configs (x-caddy) defined in other -// service specs are validated and appended to the generated Caddyfile. Invalid configs are logged and skipped to ensure -// the generated Caddyfile remains valid. +// Generate creates a Caddyfile for the local Caddy container from the supplied healthy application containers. +// Application service ports provide the generated sites. The Caddy container's custom Caddy config (x-caddy), if +// defined, provides the global block even when that container is unhealthy. When application custom Caddy configs +// are included, the newest container for each service provides its config. // -// The final Caddyfile structure includes: +// The resulting Caddyfile has this structure: // -// [caddy x-caddy (global config)] -// [generated Caddyfile from all service ports] -// [service-a x-caddy] +// [caddy custom Caddy config (global)] +// [generated sites from application service ports] +// [service-a custom Caddy config] // ... -// [service-z x-caddy] +// [service-z custom Caddy config] // -// If includeCustom is false, custom Caddy configs (x-caddy) are not included in the generated Caddyfile. +// Bootstrap mode keeps the global custom Caddy config and generated sites, but omits application custom Caddy configs. +// It skips validation because Caddy may not be running yet. In this case, Caddy validates the config when it starts. +// Global template errors still stop generation. In full mode, invalid globals stop generation, while invalid +// application custom Caddy configs are logged and skipped. func (g *CaddyfileGenerator) Generate( - ctx context.Context, records []store.ContainerRecord, includeCustom bool, + ctx context.Context, + caddyCtr api.ServiceContainer, + records []store.ContainerRecord, + bootstrap bool, ) (string, error) { + records = slices.Clone(records) // Sort records by local machine first, then by service name and creation time. Placing containers on the local // machine first lets user-defined Caddy configs pair this ordering with the "first" lb_policy to always send // traffic to the same-host replica (skipping the cross-machine hop) and only fall back to remote upstreams when // the local one is unhealthy. - // The service name and creation time tiebreakers keep the generated Caddyfile stable across regenerations. + // The service name, creation time, and container ID tiebreakers keep the file stable across regenerations. slices.SortStableFunc(records, func(a, b store.ContainerRecord) int { return cmp.Or( g.localMachineRank(a.MachineID)-g.localMachineRank(b.MachineID), strings.Compare(a.Container.ServiceName(), b.Container.ServiceName()), a.Container.CreatedTime().Compare(b.Container.CreatedTime()), + strings.Compare(a.Container.ID, b.Container.ID), ) }) @@ -136,26 +148,11 @@ func (g *CaddyfileGenerator) Generate( } caddyfileHeader := fmt.Sprintf(caddyfileHeaderFmt, g.machineName, time.Now().UTC().Format(time.RFC3339)) - if !includeCustom { - return fmt.Sprintf("%s\n%s\n%s", caddyfileHeader, caddyfile, caddyfileUnavailabeFooter), nil - } - upstreams := serviceUpstreams(containers) // Track validation errors for reporting. var configErrors []string - // Find the 'caddy' service container on this machine. Use the most recent one if multiple exist. - var caddyCtr *api.ServiceContainer - for _, cr := range records { - if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName && - (caddyCtr == nil || cr.Container.CreatedTime().Compare(caddyCtr.CreatedTime()) > 0) { - caddyCtr = &cr.Container - } - } - - // If the caddy container is running on this machine and has a custom Caddy config (global), - // prepend it to the generated Caddyfile and validate it. - if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" { + if caddyCtr.ServiceSpec.CaddyConfig() != "" { // Render the custom global Caddy config as a Go template with the upstreams. tmplCtx := templateContext{ Name: caddyCtr.ServiceName(), @@ -163,23 +160,23 @@ func (g *CaddyfileGenerator) Generate( } renderedConfig, err := renderCaddyfile(tmplCtx, caddyCtr.ServiceSpec.CaddyConfig()) if err != nil { - g.log.Error("Failed to render template directives in user-defined global Caddy config, skipping it.", - "service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err) - configErrors = append(configErrors, - fmt.Sprintf("service '%s': failed to render template: %v", caddyCtr.ServiceName(), err)) - } else { - caddyfileCandidate := fmt.Sprintf("# User-defined global config from service '%s'.\n%s\n\n%s", - caddyCtr.ServiceName(), renderedConfig, caddyfile) - + return "", fmt.Errorf( + "render template directives in user-defined global Caddy config from Caddy container %s: %w", + caddyCtr.ShortID(), err) + } + caddyfileCandidate := fmt.Sprintf("# User-defined global config from service '%s'.\n%s\n\n%s", + caddyCtr.ServiceName(), renderedConfig, caddyfile) + if !bootstrap && g.validator != nil { if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil { - g.log.Error("User-defined global Caddy config is invalid, skipping it.", - "service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err) - configErrors = append(configErrors, - fmt.Sprintf("service '%s': validation failed: %v", caddyCtr.ServiceName(), err)) - } else { - caddyfile = caddyfileCandidate + return "", fmt.Errorf("validate user-defined global Caddy config from Caddy container %s: %w", + caddyCtr.ShortID(), err) } } + caddyfile = caddyfileCandidate + } + + if bootstrap { + return caddyfileHeader + bootstrapMarker + caddyCtr.ID + "\n\n" + caddyfile, nil } // There could be multiple service containers for the same service with different custom Caddy configs, for example, @@ -200,7 +197,7 @@ func (g *CaddyfileGenerator) Generate( // Append a custom Caddy config for each service to the Caddyfile and validate it. If the config for a service // is invalid, skip it but continue processing other services to ensure the Caddyfile remains valid. for _, serviceName := range sortedServiceNames { - // Skip the caddy container as we already processed it. + // Skip the caddy container as we already processed it as the global config. if serviceName == CaddyServiceName { continue } @@ -226,16 +223,22 @@ func (g *CaddyfileGenerator) Generate( caddyfileCandidate := fmt.Sprintf("%s\n# User-defined config for service '%s'.\n%s\n", caddyfile, serviceName, renderedConfig) - if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil { - g.log.Error("User-defined Caddy config for service is invalid, skipping it.", - "service", serviceName, "err", err) - configErrors = append(configErrors, fmt.Sprintf("service '%s': validation failed: %v", serviceName, err)) - } else { - caddyfile = caddyfileCandidate + if g.validator != nil { + if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil { + if _, ok := errors.AsType[*InvalidCaddyfileError](err); !ok { + return "", fmt.Errorf("validate Caddy config for service '%s': %w", serviceName, err) + } + g.log.Error("User-defined Caddy config for service is invalid, skipping it.", + "service", serviceName, "err", err) + configErrors = append(configErrors, + fmt.Sprintf("service '%s': validation failed: %v", serviceName, err)) + continue + } } + caddyfile = caddyfileCandidate } - // Append error summary as comment if there were any invalid configs. + // Keep skipped-snippet errors in the saved file so an operator can see why routes are missing after a restart. if len(configErrors) > 0 { var errorsComment strings.Builder errorsComment.WriteString("# Skipped invalid user-defined configs:\n") @@ -249,8 +252,7 @@ func (g *CaddyfileGenerator) Generate( return caddyfileHeader + "\n" + caddyfile, nil } -// localMachineRank returns 0 if the given machineID matches the local machine and 1 otherwise. -// Useful for sorting containers running locally first. +// localMachineRank is a sorting function for containers that puts running on the local machine first. func (g *CaddyfileGenerator) localMachineRank(machineID string) int { if g.machineID == machineID { return 0 diff --git a/internal/machine/caddyconfig/caddyfile_test.go b/internal/machine/caddyconfig/caddyfile_test.go index 447333df..3ec4b10d 100644 --- a/internal/machine/caddyconfig/caddyfile_test.go +++ b/internal/machine/caddyconfig/caddyfile_test.go @@ -208,10 +208,11 @@ http://app.example.com { ctx := context.Background() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Validator is not expected to be called in these tests. - generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil) + validator := NewMockCaddyfileValidator(t) + generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil) - config, err := generator.Generate(ctx, tt.containers, true) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.1", "", "", time.Now()).Container + config, err := generator.Generate(ctx, caddyCtr, tt.containers, false) if tt.wantErr { assert.Error(t, err) @@ -332,7 +333,7 @@ bad.template.com { `, }, { - name: "caddy service with invalid global config is skipped", + name: "caddy service with invalid global config aborts", containers: []store.ContainerRecord{ newContainerRecordWithCaddyConfig( "caddy", @@ -345,10 +346,7 @@ localhost { time.Now(), ), }, - want: testCaddyfileHeader + ` -# Skipped invalid user-defined configs: -# - service 'caddy': validation failed: invalid config detected -`, + wantErr: true, }, { name: "caddy service on different machine is ignored", @@ -786,7 +784,7 @@ web-v2.example.com { `, }, { - name: "multiple errors: invalid global, template error, and validation error", + name: "invalid global aborts before application configs", containers: []store.ContainerRecord{ newContainerRecordWithCaddyConfig( "caddy", @@ -827,17 +825,7 @@ invalid.example.com { time.Now(), ), }, - want: testCaddyfileHeader + ` -# User-defined config for service 'valid'. -valid.example.com { - respond "Valid config" -} - -# Skipped invalid user-defined configs: -# - service 'caddy': validation failed: invalid config detected -# - service 'broken-template': failed to render template: parse config as Go template: template: Caddyfile:2: unterminated quoted string -# - service 'invalid': validation failed: invalid config detected -`, + wantErr: true, }, } @@ -846,7 +834,7 @@ valid.example.com { validator.EXPECT().Validate(mock.Anything, mock.Anything).RunAndReturn( func(ctx context.Context, caddyfile string) error { if strings.Contains(caddyfile, "# test:invalid") { - return errors.New("invalid config detected") + return &InvalidCaddyfileError{Message: "invalid config detected"} } return nil }) @@ -855,7 +843,8 @@ valid.example.com { t.Run(tt.name, func(t *testing.T) { generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil) - config, err := generator.Generate(ctx, tt.containers, true) + caddyCtr := caddyContainerFromTestContainers(tt.containers) + config, err := generator.Generate(ctx, caddyCtr, tt.containers, false) if tt.wantErr { assert.Error(t, err) @@ -868,6 +857,167 @@ valid.example.com { } } +func TestCaddyfileGeneratorBootstrap(t *testing.T) { + // Bootstrap keeps the local Caddy container's globals and generated routes, but omits application custom configs. + tests := []struct { + name string + containers []store.ContainerRecord + want string + }{ + { + name: "global config retained and application custom configs skipped", + containers: []store.ContainerRecord{ + newContainerRecordWithCaddyConfig( + "caddy", + "10.210.0.1", + `# Global config +{ + global directive +}`, + "test-machine-id", + time.Now(), + ), + newContainerRecordWithCaddyConfig( + "web", + "10.210.0.2", + `web.example.com { + reverse_proxy web:3000 +}`, + "test-machine-id", + time.Now(), + ), + newContainerRecordWithPorts( + "api", + "10.210.0.3", + []string{"api.example.com:8080/http"}, + "test-machine-id", + ), + }, + want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", ` +# Uncloud bootstrap for Caddy container caddy-10.210.0.1 + +# User-defined global config from service 'caddy'. +# Global config +{ + global directive +} + +# Health check endpoint`, 1) + ` +# Sites generated from service ports. + +http://api.example.com { + reverse_proxy 10.210.0.3:8080 { + import common_proxy + } + log +} +`, + }, + { + name: "no containers with x-caddy configs", + containers: []store.ContainerRecord{ + newContainerRecordWithPorts( + "api", + "10.210.0.3", + []string{"api.example.com:8080/http"}, + "test-machine-id", + ), + }, + want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", ` +# Uncloud bootstrap for Caddy container caddy-10.210.0.1 + +# Health check endpoint`, 1) + ` +# Sites generated from service ports. + +http://api.example.com { + reverse_proxy 10.210.0.3:8080 { + import common_proxy + } + log +} +`, + }, + } + + ctx := context.Background() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + validator := NewMockCaddyfileValidator(t) + generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil) + + caddyContainer := caddyContainerFromTestContainers(tt.containers) + config, err := generator.Generate(ctx, caddyContainer, tt.containers, true) + require.NoError(t, err) + + assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match") + }) + } +} + +func TestCaddyfileGenerator_ValidatorTransportErrorAbortsGeneration(t *testing.T) { + validator := NewMockCaddyfileValidator(t) + validator.EXPECT().Validate(mock.Anything, mock.Anything).Return(errors.New("socket disappeared")) + + generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil) + app := newContainerRecordWithCaddyConfig( + "web", + "10.210.0.2", + "web.example.com { respond ok }", + "test-machine-id", + time.Now(), + ) + + _, err := generator.Generate(context.Background(), caddyContainerFromTestContainers(nil), + []store.ContainerRecord{app}, false) + require.ErrorContains(t, err, "socket disappeared") +} + +func newContainer(ip string, ports ...string) api.ServiceContainer { + portsLabel := strings.Join(ports, ",") + return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{ + ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Running: true, + }, + }, + NetworkSettings: &container.NetworkSettings{ + Networks: map[string]*network.EndpointSettings{ + docker.NetworkName: { + IPAddress: ip, + }, + }, + }, + Config: &container.Config{ + Labels: map[string]string{ + api.LabelServicePorts: portsLabel, + }, + }, + }}} +} + +func newContainerWithoutNetwork(ports ...string) api.ServiceContainer { + portsLabel := strings.Join(ports, ",") + return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{ + ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{ + Running: true, + }, + }, + NetworkSettings: &container.NetworkSettings{ + Networks: map[string]*network.EndpointSettings{ + "other-network": { + IPAddress: "172.17.0.2", + }, + }, + }, + Config: &container.Config{ + Labels: map[string]string{ + api.LabelServicePorts: portsLabel, + }, + }, + }}} +} + func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord { return store.ContainerRecord{ Container: ctr, @@ -911,100 +1061,6 @@ func newContainerRecordWithCaddyConfig(serviceName, ip, caddyConfig, machineID s } } -func TestCaddyfileGeneratorWithoutCustomConfigs(t *testing.T) { - // Test that when includeCustom is false (Caddy not available), x-caddy configs are skipped. - tests := []struct { - name string - containers []store.ContainerRecord - want string - }{ - { - name: "x-caddy configs are skipped", - containers: []store.ContainerRecord{ - newContainerRecordWithCaddyConfig( - "caddy", - "10.210.0.1", - `# Global config -{ - global directive -}`, - "test-machine-id", - time.Now(), - ), - newContainerRecordWithCaddyConfig( - "web", - "10.210.0.2", - `web.example.com { - reverse_proxy web:3000 -}`, - "test-machine-id", - time.Now(), - ), - newContainerRecordWithPorts( - "api", - "10.210.0.3", - []string{"api.example.com:8080/http"}, - "test-machine-id", - ), - }, - want: testCaddyfileHeader + ` -# Sites generated from service ports. - -http://api.example.com { - reverse_proxy 10.210.0.3:8080 { - import common_proxy - } - log -} - -# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine -# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest -# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy) -# and its logs for more details (uc logs caddy). -`, - }, - { - name: "no containers with x-caddy configs", - containers: []store.ContainerRecord{ - newContainerRecordWithPorts( - "api", - "10.210.0.3", - []string{"api.example.com:8080/http"}, - "test-machine-id", - ), - }, - want: testCaddyfileHeader + ` -# Sites generated from service ports. - -http://api.example.com { - reverse_proxy 10.210.0.3:8080 { - import common_proxy - } - log -} - -# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine -# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest -# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy) -# and its logs for more details (uc logs caddy). -`, - }, - } - - ctx := context.Background() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Validator is not expected to be called in these tests. - generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil) - - config, err := generator.Generate(ctx, tt.containers, false) - require.NoError(t, err) - - assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match") - }) - } -} - func newContainerRecordWithPorts(serviceName, ip string, ports []string, machineID string) store.ContainerRecord { portsLabel := strings.Join(ports, ",") return store.ContainerRecord{ @@ -1037,3 +1093,18 @@ func newContainerRecordWithPorts(serviceName, ip string, ports []string, machine MachineID: machineID, } } + +// caddyContainerFromTestContainers returns the local Caddy container from the provided test containers, +// or creates a default one if not found. +func caddyContainerFromTestContainers(records []store.ContainerRecord) api.ServiceContainer { + if caddyCtr := selectLocalCaddyContainer(records, "test-machine-id"); caddyCtr != nil { + return *caddyCtr + } + return newContainerRecordWithCaddyConfig( + "caddy", + "10.210.0.1", + "", + "test-machine-id", + time.Now(), + ).Container +} diff --git a/internal/machine/caddyconfig/client.go b/internal/machine/caddyconfig/client.go index 2901a632..f1860735 100644 --- a/internal/machine/caddyconfig/client.go +++ b/internal/machine/caddyconfig/client.go @@ -3,7 +3,6 @@ package caddyconfig import ( "context" "encoding/json" - "errors" "fmt" "io" "net" @@ -26,15 +25,15 @@ func NewCaddyAdminClient(socketPath string) *CaddyAdminClient { client: &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", socketPath) + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) }, }, }, } } -// IsAvailable checks if the local Caddy instance is listening on the admin socket. +// IsAvailable checks whether a Caddy process accepts connections at the admin socket. func (c *CaddyAdminClient) IsAvailable() bool { conn, err := net.DialTimeout("unix", c.socketPath, 1*time.Second) // A stale socket file left over from a crashed Caddy container returns ECONNREFUSED so this is correctly handled @@ -80,15 +79,16 @@ func (c *CaddyAdminClient) Adapt(ctx context.Context, caddyfile string) (string, // If the response is a 400 Bad Request, try to parse the error message from it. if resp.StatusCode == http.StatusBadRequest { var apiError caddy.APIError - if err = json.Unmarshal(body, &apiError); err == nil { - return "", errors.New(apiError.Message) + if err = json.Unmarshal(body, &apiError); err == nil && apiError.Message != "" { + return "", &InvalidCaddyfileError{Message: apiError.Message} } + return "", &InvalidCaddyfileError{Message: string(body)} } - return "", errors.New(string(body)) + return "", fmt.Errorf("adapt request failed: HTTP %d: %s", resp.StatusCode, string(body)) } -// Load loads a Caddyfile configuration into the Caddy instance running on the machine. +// Load loads a Caddyfile configuration into the Caddy instance. // Due to a Caddy bug (https://github.com/caddyserver/caddy/issues/7246), we first adapt the Caddyfile to JSON // and then load the JSON config to get proper error handling. func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error { @@ -96,7 +96,11 @@ func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error { if err != nil { return fmt.Errorf("adapt Caddyfile to JSON config: %w", err) } + return c.LoadJSON(ctx, jsonConfig) +} +// LoadJSON loads a JSON configuration into the Caddy instance. +func (c *CaddyAdminClient) LoadJSON(ctx context.Context, jsonConfig string) error { req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/load", strings.NewReader(jsonConfig)) if err != nil { return fmt.Errorf("create load request: %w", err) diff --git a/internal/machine/caddyconfig/client_test.go b/internal/machine/caddyconfig/client_test.go new file mode 100644 index 00000000..790800cb --- /dev/null +++ b/internal/machine/caddyconfig/client_test.go @@ -0,0 +1,54 @@ +package caddyconfig + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testAdminServer(t *testing.T, handler http.HandlerFunc) string { + t.Helper() + // macOS limits Unix socket paths to 104 bytes. t.TempDir can exceed that. + dir, err := os.MkdirTemp("/tmp", "uc-caddy-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(dir) + }) + + socketPath := filepath.Join(dir, "admin.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + server := httptest.NewUnstartedServer(handler) + _ = server.Listener.Close() + server.Listener = listener + server.Start() + t.Cleanup(server.Close) + + return socketPath +} + +func TestCaddyAdminClient_Validate(t *testing.T) { + var status atomic.Int64 + status.Store(http.StatusBadRequest) + socket := testAdminServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(int(status.Load())) + _, _ = w.Write([]byte(`{"message":"invalid config"}`)) + }) + client := NewCaddyAdminClient(socket) + err := client.Validate(context.Background(), "invalid") + var invalid *InvalidCaddyfileError + assert.ErrorAs(t, err, &invalid) + + status.Store(http.StatusInternalServerError) + err = client.Validate(context.Background(), "invalid") + assert.Error(t, err) + assert.NotErrorAs(t, err, &invalid) +} diff --git a/internal/machine/caddyconfig/controller.go b/internal/machine/caddyconfig/controller.go index d16863d7..903a1c3b 100644 --- a/internal/machine/caddyconfig/controller.go +++ b/internal/machine/caddyconfig/controller.go @@ -2,7 +2,7 @@ package caddyconfig import ( "context" - "encoding/json" + "errors" "fmt" "log/slog" "net/netip" @@ -10,6 +10,7 @@ import ( "path/filepath" "slices" "strings" + "time" "github.com/psviderski/uncloud/internal/fs" "github.com/psviderski/uncloud/internal/machine/store" @@ -20,27 +21,40 @@ const ( CaddyServiceName = "caddy" CaddyGroup = "uncloud" VerifyPath = "/.uncloud-verify" + // periodicReconciliationInterval is the interval at which the controller reconciles the Caddyfile + // even if no container changes are observed. + periodicReconciliationInterval = 30 * time.Second ) -// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse -// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal -// network. +// Controller keeps the local Caddy reverse proxy in sync with container state from the cluster store. It writes a +// Caddyfile that routes external traffic to healthy service containers across the internal network. +// +// Current Caddy deployments share one Caddyfile and one admin socket per machine. The controller must preserve a +// saved full Caddyfile when Caddy is unavailable because that file may be needed to restart or roll back a container. +// It writes a reduced bootstrap config only when no full file exists, including when an older offline file must be +// replaced. The shared layout cannot give overlapping Caddy revisions separate configurations. type Controller struct { machineID string caddyfilePath string + service *Service generator *CaddyfileGenerator client *CaddyAdminClient store *store.Store log *slog.Logger - // lastFingerprint caches the fingerprint of the containers used to generate the latest successfully loaded - // Caddyfile. nil means it hasn't been loaded yet or the last load failed. + // Tests use their own group so they can exercise real file publication without the daemon's uncloud group. + fileGroup string + // lastFingerprint records the healthy application-container inputs used for the last full Caddyfile that was + // successfully loaded into Caddy and saved to disk. It excludes the Caddy container, whose identity, start time, + // and global config are tracked separately below. lastFingerprint []containerFingerprint - // lastCaddyfile caches the last generated Caddyfile. - lastCaddyfile string + lastCaddyCtrID string + lastStartedAt string + lastGlobal string + lastSavedBody string } -// containerFingerprint is the subset of container data that the Caddyfile generator depends on. -// Comparing fingerprints lets the controller skip no-op regenerations. +// containerFingerprint contains container identity and routing inputs that can change the generated Caddyfile. +// It excludes incidental Docker state so unrelated container updates do not cause a Caddy reload. type containerFingerprint struct { ID string IP netip.Addr @@ -56,7 +70,8 @@ func (f containerFingerprint) Equal(other containerFingerprint) bool { f.CaddyConfig == other.CaddyConfig } -func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) { +func NewController(machineID string, service *Service, adminSock string, store *store.Store) (*Controller, error) { + configDir := service.configDir if err := os.MkdirAll(configDir, 0o750); err != nil { return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err) } @@ -67,16 +82,21 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) ( log := slog.With("component", "caddy-controller") client := NewCaddyAdminClient(adminSock) - // generator is initialised by Run() once the machine name is resolved from the store. + // The generator is initialised by Run() after resolving the machine name from the store. return &Controller{ machineID: machineID, caddyfilePath: filepath.Join(configDir, "Caddyfile"), + service: service, client: client, store: store, log: log, + fileGroup: CaddyGroup, }, nil } +// Run reconciles on container changes and every periodicReconciliationInterval. Failed work is retried every second, +// but repeated retry failures are logged only during the periodic check. +// The periodic check also covers missed events and outstanding failures after the daemon restarts. func (c *Controller) Run(ctx context.Context) error { // Default the machine name to the machine ID so the Caddyfile header still carries a stable identifier if // the store lookup fails. @@ -95,14 +115,35 @@ func (c *Controller) Run(ctx context.Context) error { } c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.") - containers = filterHealthyContainers(containers) - c.generateAndLoadCaddyfile(ctx, containers) + var retryC <-chan time.Time + periodic := time.NewTicker(periodicReconciliationInterval) + defer periodic.Stop() - // TODO: left for backward compatibility, remove later. - if err = c.generateJSONConfig(containers); err != nil { - c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err) + handleResult := func(err error, logFailure bool) { + c.service.setReconciliationResult(err) + if err == nil { + retryC = nil + return + } + if logFailure { + c.log.Error("Failed to reconcile Caddy configuration, will retry.", "err", err) + } + retryC = time.After(time.Second) } + reconcile := func(logFailure bool) { + ctrs, err := c.store.ListContainers(ctx, store.ListOptions{}) + if err != nil { + err = fmt.Errorf("list containers: %w", err) + } else { + err = c.generateAndLoadCaddyfile(ctx, ctrs) + } + handleResult(err, logFailure) + } + + // The subscription supplies an initial snapshot, so the first attempt need not wait for a change event. + handleResult(c.generateAndLoadCaddyfile(ctx, containers), true) + for { select { case _, ok := <-changes: @@ -111,25 +152,18 @@ func (c *Controller) Run(ctx context.Context) error { } c.log.Debug("Cluster containers changed, regenerating Caddy configuration.") - containers, err = c.store.ListContainers(ctx, store.ListOptions{}) - if err != nil { - c.log.Error("Failed to list containers.", "err", err) - continue - } - containers = filterHealthyContainers(containers) - c.generateAndLoadCaddyfile(ctx, containers) - - // TODO: left for backward compatibility, remove later. - if err = c.generateJSONConfig(containers); err != nil { - c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err) - } + reconcile(true) + case <-periodic.C: + reconcile(true) + case <-retryC: + reconcile(false) case <-ctx.Done(): return nil } } } -// filterHealthyContainers filters out unhealthy and hook containers. +// filterHealthyContainers filters out unhealthy, hook, and caddy containers. // TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined // by the cluster membership state of the machine that the container is running on. Implement machine membership // check using Corrossion Admin client. @@ -139,6 +173,9 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain if cr.Container.IsHook() { continue } + if cr.Container.ServiceName() == CaddyServiceName { + continue + } if cr.Container.Healthy() { healthy = append(healthy, cr) } @@ -146,65 +183,96 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain return healthy } -// generateAndLoadCaddyfile regenerates the Caddyfile from the given containers and loads it into the local Caddy -// if available. -func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) { - // Check if Caddy is available before attempting to generate and load config. - caddyAvailable := c.client.IsAvailable() - - // Skip regeneration when Caddy is available and the containers since the last successful load haven't changed. - // When Caddy is unavailable we still regenerate to keep the Caddyfile on disk updated. - fingerprint := fingerprintContainers(containers) - if caddyAvailable && slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) { - c.log.Debug("Caddy configuration is unchanged.", "path", c.caddyfilePath) - return +// generateAndLoadCaddyfile reconciles the shared Caddyfile for the selected local Caddy container. A full candidate +// reaches disk only after Caddy accepts it. Without an admin endpoint, the controller keeps a saved full file intact +// and writes a bootstrap config only when the file is absent or already a bootstrap. +func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) error { + caddyCtr := selectLocalCaddyContainer(containers, c.machineID) + if caddyCtr == nil { + // Caddy is not running locally which means there is no reliable source for the global config, skipping. + return nil } - caddyfile, err := c.generator.Generate(ctx, containers, caddyAvailable) - if err != nil { - c.log.Error("Failed to generate Caddyfile configuration.", "err", err) - return + saved, readErr := os.ReadFile(c.caddyfilePath) + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return fmt.Errorf("read saved Caddyfile: %w", readErr) } + haveSaved := readErr == nil - if !caddyAvailable { - // Caddy is not running so the generated Caddyfile should not include user-defined configs thus must be valid. - // It's safe to write the config to disk so that when Caddy is deployed on this machine, it can pick it up. - if err = c.writeCaddyfileIfChanged(caddyfile); err != nil { - c.log.Error("Failed to write Caddyfile to disk.", "err", err) - return + healthyCtrs := filterHealthyContainers(containers) + fingerprint := fingerprintContainers(healthyCtrs) + + if !caddyCtr.State.Running || !c.client.IsAvailable() { + if haveSaved && !isBootstrapCaddyfile(string(saved)) { + // Caddy may need this file to restart or roll back. A hosts-only replacement could drop + // storage or other global settings while the admin API is unavailable. + if caddyCtr.State.Running { + return fmt.Errorf("caddy admin socket unavailable, preserving saved Caddyfile") + } + return nil } - c.log.Debug("Caddy is not running on this machine, skipping configuration load.", "path", c.caddyfilePath) - return + + bootstrap, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, true) + if err != nil { + return fmt.Errorf("generate Caddy bootstrap config: %w", err) + } + if err = c.writeCaddyfileIfChanged(bootstrap); err != nil { + return fmt.Errorf("save Caddy bootstrap config: %w", err) + } + if caddyCtr.State.Running { + return fmt.Errorf("caddy admin socket unavailable, bootstrap config saved") + } + return nil } - // Caddy is available, try to load the config which may fail if the config is invalid. Generally, a config can - // pass the adaptation/validation step but still fail to load, for example, if it references resources that are - // not available. + if haveSaved && + caddyfileBody(string(saved)) == c.lastSavedBody && + !isBootstrapCaddyfile(string(saved)) && + c.lastCaddyCtrID == caddyCtr.ID && + c.lastStartedAt == caddyCtr.State.StartedAt && + c.lastGlobal == caddyCtr.ServiceSpec.CaddyConfig() && + slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) { + // No changes to the inputs that affect the generated Caddyfile, so no reload is needed. + return nil + } + + caddyfile, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, false) + if err != nil { + return fmt.Errorf("generate Caddyfile: %w", err) + } + // Try to load the config which may fail if the config is invalid. Generally, a config can pass + // the adaptation/validation step but still fail to load, for example, if it references resources + // that are not available. if err = c.client.Load(ctx, caddyfile); err != nil { - c.log.Error("Failed to load new Caddy configuration into local Caddy instance.", - "err", err, "path", c.caddyfilePath) - // Mark the cache stale so the next container change retries the load even if the container set is unchanged. - c.lastFingerprint = nil - // Don't write invalid config to disk. - return + return fmt.Errorf("load Caddyfile: %w", err) + } + if err = c.writeCaddyfileIfChanged(caddyfile); err != nil { + return fmt.Errorf("save loaded Caddyfile: %w", err) } c.lastFingerprint = fingerprint - - // Config loaded successfully, now write it to disk. - if err = c.writeCaddyfileIfChanged(caddyfile); err != nil { - c.log.Error("Failed to write Caddyfile to disk after successful load.", "err", err) - // Config is already loaded in Caddy, so this is not critical. The next regeneration retries the disk write. - return - } - + c.lastCaddyCtrID = caddyCtr.ID + c.lastStartedAt = caddyCtr.State.StartedAt + c.lastGlobal = caddyCtr.ServiceSpec.CaddyConfig() + c.lastSavedBody = caddyfileBody(caddyfile) c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath) + + return nil +} + +// isBootstrapCaddyfile distinguishes a reduced startup file from a full file that must survive Caddy outage. +// It also recognizes the older daemon's offline file. That file omitted every x-caddy block, including globals, +// so treating it as a protected full file would preserve the configuration-loss bug: +// https://github.com/psviderski/uncloud/issues/412 +func isBootstrapCaddyfile(caddyfile string) bool { + return strings.Contains(caddyfile, bootstrapMarker) || strings.Contains(caddyfile, legacyBootstrapMarker) } // fingerprintContainers returns a fingerprint of containers that the Caddyfile generator depends on. func fingerprintContainers(containers []store.ContainerRecord) []containerFingerprint { fingerprints := make([]containerFingerprint, len(containers)) for i, cr := range containers { - // Ignore ports parsing error as not much we can do about it. The generator just logs them and continues. + // Ignore ports parsing error as not much we can do about it. The generator just logs them and skips + // the container. ports, _ := cr.Container.ServicePorts() fingerprints[i] = containerFingerprint{ ID: cr.Container.ID, @@ -220,21 +288,71 @@ func fingerprintContainers(containers []store.ContainerRecord) []containerFinger return fingerprints } -// writeCaddyfileIfChanged writes the Caddyfile content to disk with proper permissions only if its body differs -// from the last successfully written content. The first line of the Caddyfile carries a generation timestamp that -// changes on every regeneration, so it's excluded from the comparison to avoid redundant writes. +// selectLocalCaddyContainer chooses which local Caddy container supplies global Caddy config. A running container +// takes precedence over a newer stopped replacement, even if the running container is unhealthy. If two revisions +// run at once, the shared admin socket does not identify its owner, so this choice remains best effort. +func selectLocalCaddyContainer(records []store.ContainerRecord, machineID string) *api.ServiceContainer { + var selected *api.ServiceContainer + for _, cr := range records { + ctr := cr.Container + if cr.MachineID != machineID || ctr.ServiceName() != CaddyServiceName || ctr.IsHook() { + continue + } + if selected == nil { + selected = &ctr + continue + } + if ctr.State.Running != selected.State.Running { + if ctr.State.Running { + selected = &ctr + } + continue + } + if ctr.CreatedTime().Compare(selected.CreatedTime()) > 0 { + selected = &ctr + } + } + + return selected +} + +// writeCaddyfileIfChanged atomically replaces the saved Caddyfile only if its body differs from the new content. func (c *Controller) writeCaddyfileIfChanged(caddyfile string) error { - if caddyfileBody(caddyfile) == caddyfileBody(c.lastCaddyfile) { + saved, err := os.ReadFile(c.caddyfilePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("read Caddyfile '%s': %w", c.caddyfilePath, err) + } + if err == nil && caddyfileBody(caddyfile) == caddyfileBody(string(saved)) { return nil } - if err := os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil { - return fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err) + dir := filepath.Dir(c.caddyfilePath) + tmp, err := os.CreateTemp(dir, ".Caddyfile-*") + if err != nil { + return fmt.Errorf("create temporary Caddyfile: %w", err) } - if err := fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil { - return fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err) + defer os.Remove(tmp.Name()) + defer tmp.Close() + + if err = tmp.Chmod(0o640); err != nil { + return fmt.Errorf("set temporary Caddyfile permissions: %w", err) + } + if err = fs.Chown(tmp.Name(), "", c.fileGroup); err != nil { + return fmt.Errorf("change owner of temporary Caddyfile: %w", err) + } + + if _, err = tmp.WriteString(caddyfile); err != nil { + return fmt.Errorf("write temporary Caddyfile: %w", err) + } + if err = tmp.Sync(); err != nil { + return fmt.Errorf("sync temporary Caddyfile: %w", err) + } + if err = tmp.Close(); err != nil { + return fmt.Errorf("close temporary Caddyfile: %w", err) + } + if err = os.Rename(tmp.Name(), c.caddyfilePath); err != nil { + return fmt.Errorf("replace Caddyfile '%s': %w", c.caddyfilePath, err) } - c.lastCaddyfile = caddyfile return nil } @@ -247,30 +365,3 @@ func caddyfileBody(caddyfile string) string { } return caddyfile } - -func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error { - serviceContainers := make([]api.ServiceContainer, len(containers)) - for i, cr := range containers { - serviceContainers[i] = cr.Container - } - - config, err := GenerateJSONConfig(serviceContainers, c.machineID) - if err != nil { - return err - } - - configBytes, err := json.MarshalIndent(config, "", " ") - if err != nil { - return fmt.Errorf("marshal Caddy configuration: %w", err) - } - configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json") - - if err = os.WriteFile(configPath, configBytes, 0o640); err != nil { - return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err) - } - if err = fs.Chown(configPath, "", CaddyGroup); err != nil { - return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err) - } - - return nil -} diff --git a/internal/machine/caddyconfig/controller_test.go b/internal/machine/caddyconfig/controller_test.go index 86135258..1e4f49b6 100644 --- a/internal/machine/caddyconfig/controller_test.go +++ b/internal/machine/caddyconfig/controller_test.go @@ -1,14 +1,488 @@ package caddyconfig import ( + "context" + "io" + "log/slog" + "net/http" "net/netip" + "os" + "os/user" + "path/filepath" "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" "testing" + "time" + "github.com/docker/docker/api/types/container" + "github.com/psviderski/uncloud/internal/machine/store" "github.com/psviderski/uncloud/pkg/api" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func testController(t *testing.T, socketPath string) *Controller { + t.Helper() + group, err := user.LookupGroupId(strconv.Itoa(os.Getegid())) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "Caddyfile") + client := NewCaddyAdminClient(socketPath) + return &Controller{ + machineID: "test-machine-id", + caddyfilePath: path, + generator: NewCaddyfileGenerator("test-machine-id", "test-machine", client, nil), + client: client, + log: slog.Default(), + fileGroup: group.Name, + } +} + +type testCaddyAdmin struct { + mu sync.Mutex + adaptRequests []string + loadCount int + rejectAdaptContaining string + rejectLoad bool +} + +func (a *testCaddyAdmin) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/adapt": + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + a.mu.Lock() + a.adaptRequests = append(a.adaptRequests, string(body)) + reject := a.rejectAdaptContaining != "" && strings.Contains(string(body), a.rejectAdaptContaining) + a.mu.Unlock() + if reject { + http.Error(w, "invalid Caddyfile", http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, `{"result":{}}`) + case "/load": + a.mu.Lock() + a.loadCount++ + reject := a.rejectLoad + a.mu.Unlock() + if reject { + http.Error(w, "load rejected", http.StatusBadRequest) + } + default: + http.NotFound(w, r) + } +} + +func (a *testCaddyAdmin) snapshot() ([]string, int) { + a.mu.Lock() + defer a.mu.Unlock() + return append([]string(nil), a.adaptRequests...), a.loadCount +} + +func (a *testCaddyAdmin) setRejectLoad(reject bool) { + a.mu.Lock() + a.rejectLoad = reject + a.mu.Unlock() +} + +func TestController_GenerateAndLoadCaddyfile(t *testing.T) { + t.Run("keeps the saved file when there is no local Caddy container", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + saved := "# previous full Caddyfile\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ remote }", + "other-machine", time.Now()) + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{remote})) + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + }) + + t.Run("does not publish a bootstrap with an invalid global template", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{{upstreams", + "test-machine-id", time.Now()) + + require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), + []store.ContainerRecord{caddyCtr}), "render template") + _, err := os.Stat(controller.caddyfilePath) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("bootstraps unhealthy Caddy with its globals and healthy routes", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"} + newerStopped := newContainerRecordWithCaddyConfig("caddy", "10.210.0.6", "{\n\tstorage replacement\n}", + "test-machine-id", time.Now().Add(time.Minute)) + newerStopped.Container.State.Running = false + app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"}, + "test-machine-id") + app.Container.ServiceSpec.Caddy = &api.CaddySpec{Config: "custom.example.com { respond app }"} + unhealthy := newContainerRecordWithPorts("unhealthy", "10.210.0.4", + []string{"unhealthy.example.com:8080/http"}, "test-machine-id") + unhealthy.Container.State.Health = &container.Health{Status: "unhealthy"} + hook := newContainerRecordWithPorts("hook", "10.210.0.5", + []string{"hook.example.com:8080/http"}, "test-machine-id") + hook.Container.Config.Labels[api.LabelHook] = "pre-deploy" + + err := controller.generateAndLoadCaddyfile(context.Background(), + []store.ContainerRecord{newerStopped, caddyCtr, app, unhealthy, hook}) + require.ErrorContains(t, err, "admin socket unavailable") + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID) + assert.Contains(t, string(saved), "storage uncloud") + assert.NotContains(t, string(saved), "storage replacement") + assert.Contains(t, string(saved), "app.example.com") + assert.NotContains(t, string(saved), "custom.example.com") + assert.NotContains(t, string(saved), "unhealthy.example.com") + assert.NotContains(t, string(saved), "hook.example.com") + }) + + t.Run("replaces a legacy offline file with a bootstrap containing globals", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + legacy := "# generated by older daemon\nhttp:// { respond ok }\n" + legacyBootstrapMarker + "\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(legacy), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + + err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr}) + require.ErrorContains(t, err, "admin socket unavailable") + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), "storage uncloud") + assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID) + assert.NotContains(t, string(saved), legacyBootstrapMarker) + }) + + t.Run("saves a bootstrap for a stopped Caddy container with no file", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + caddyCtr.Container.State.Running = false + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr})) + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID) + assert.Contains(t, string(saved), "storage uncloud") + }) + + for _, tc := range []struct { + name string + running bool + }{ + {name: "preserves a full file when the admin socket is unavailable", running: true}, + {name: "preserves a stopped container's full file", running: false}, + } { + t.Run(tc.name, func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage new\n}", + "test-machine-id", time.Now()) + caddyCtr.Container.State.Running = tc.running + app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"new.example.com:8080/http"}, + "test-machine-id") + + err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app}) + if tc.running { + require.ErrorContains(t, err, "admin socket unavailable") + } else { + require.NoError(t, err) + } + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + }) + } + + t.Run("loads a full file and reloads after Caddy restarts", func(t *testing.T) { + admin := &testCaddyAdmin{} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"} + app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "app.example.com { respond app }", + "test-machine-id", time.Now()) + records := []store.ContainerRecord{caddyCtr, app} + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), "storage uncloud") + assert.Contains(t, string(saved), "app.example.com") + assert.NotContains(t, string(saved), bootstrapMarker) + adapted, loads := admin.snapshot() + require.NotEmpty(t, adapted) + assert.Contains(t, adapted[0], "storage uncloud") + assert.Equal(t, 1, loads) + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + _, loads = admin.snapshot() + assert.Equal(t, 1, loads, "unchanged inputs should not reload Caddy") + + records[0].Container.State.StartedAt = time.Now().UTC().Format(time.RFC3339Nano) + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + _, loads = admin.snapshot() + assert.Equal(t, 2, loads, "a restarted Caddy process needs the full configuration") + }) + + t.Run("reloads when an application route changes", func(t *testing.T) { + admin := &testCaddyAdmin{} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"}, + "test-machine-id") + records := []store.ContainerRecord{caddyCtr, app} + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + _, loads := admin.snapshot() + assert.Equal(t, 1, loads) + + records[1] = newContainerRecordWithPorts("web", "10.210.0.4", + []string{"app.example.com:8080/http"}, "test-machine-id") + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), "10.210.0.4:8080") + assert.NotContains(t, string(saved), "10.210.0.3:8080") + _, loads = admin.snapshot() + assert.Equal(t, 2, loads) + }) + + t.Run("rejects invalid globals without loading or replacing the saved file", func(t *testing.T) { + admin := &testCaddyAdmin{rejectAdaptContaining: "storage rejected"} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage rejected\n}", + "test-machine-id", time.Now()) + + err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr}) + require.ErrorContains(t, err, "validate user-defined global Caddy config") + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + _, loads := admin.snapshot() + assert.Zero(t, loads) + }) + + t.Run("skips invalid application custom config", func(t *testing.T) { + admin := &testCaddyAdmin{rejectAdaptContaining: "invalid.example.com"} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + invalid := newContainerRecordWithCaddyConfig("invalid", "10.210.0.3", + "invalid.example.com { respond bad }", "test-machine-id", time.Now()) + valid := newContainerRecordWithCaddyConfig("valid", "10.210.0.4", + "valid.example.com { respond good }", "test-machine-id", time.Now()) + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), + []store.ContainerRecord{caddyCtr, invalid, valid})) + saved, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(saved), "storage uncloud") + assert.Contains(t, string(saved), "valid.example.com") + assert.NotContains(t, string(saved), "invalid.example.com") + assert.Contains(t, string(saved), "Skipped invalid user-defined configs") + _, loads := admin.snapshot() + assert.Equal(t, 1, loads) + }) + + t.Run("keeps the saved file when application validation cannot complete", func(t *testing.T) { + var loads atomic.Int64 + socket := testAdminServer(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/adapt": + http.Error(w, "adapter unavailable", http.StatusInternalServerError) + case "/load": + loads.Add(1) + default: + http.NotFound(w, r) + } + }) + controller := testController(t, socket) + saved := "old.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "", + "test-machine-id", time.Now()) + app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "new.example.com { respond new }", + "test-machine-id", time.Now()) + + err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app}) + require.ErrorContains(t, err, "validate Caddy config for service") + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + assert.Zero(t, loads.Load()) + }) + + t.Run("retries a rejected load without changing inputs", func(t *testing.T) { + admin := &testCaddyAdmin{rejectLoad: true} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + records := []store.ContainerRecord{caddyCtr} + + require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "load Caddyfile") + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + admin.setRejectLoad(false) + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + got, err = os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(got), "storage uncloud") + _, loads := admin.snapshot() + assert.Equal(t, 2, loads) + }) + + t.Run("retries saving after a successful load", func(t *testing.T) { + admin := &testCaddyAdmin{} + controller := testController(t, testAdminServer(t, admin.ServeHTTP)) + group := controller.fileGroup + controller.fileGroup = "uncloud-test-group-that-does-not-exist" + saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + records := []store.ContainerRecord{caddyCtr} + + require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "save loaded Caddyfile") + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + controller.fileGroup = group + + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + got, err = os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(got), "storage uncloud") + _, loads := admin.snapshot() + assert.Equal(t, 2, loads) + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + _, loads = admin.snapshot() + assert.Equal(t, 2, loads, "a loaded and saved revision should not reload") + }) + + t.Run("loads a saved bootstrap when the admin socket becomes available", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}", + "test-machine-id", time.Now()) + records := []store.ContainerRecord{caddyCtr} + require.Error(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + bootstrap, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Contains(t, string(bootstrap), bootstrapMarker) + + admin := &testCaddyAdmin{} + socket := testAdminServer(t, admin.ServeHTTP) + controller.client = NewCaddyAdminClient(socket) + controller.generator = NewCaddyfileGenerator("test-machine-id", "test-machine", controller.client, nil) + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + full, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.NotContains(t, string(full), bootstrapMarker) + _, loads := admin.snapshot() + assert.Equal(t, 1, loads) + + require.NoError(t, os.WriteFile(controller.caddyfilePath, bootstrap, 0o640)) + require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records)) + _, loads = admin.snapshot() + assert.Equal(t, 2, loads, "a saved bootstrap is not an applied full configuration") + }) +} + +func TestController_WriteCaddyfileIfChanged(t *testing.T) { + t.Run("publishes a complete file with the configured group", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + caddyfile := "# Generated now\n{\n\tstorage uncloud\n}\n" + + require.NoError(t, controller.writeCaddyfileIfChanged(caddyfile)) + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, caddyfile, string(got)) + info, err := os.Stat(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o640), info.Mode().Perm()) + }) + + t.Run("does not replace a file when only the timestamp changes", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + saved := "# Generated yesterday\nsite.example.com { respond ok }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + before, err := os.Stat(controller.caddyfilePath) + require.NoError(t, err) + + require.NoError(t, controller.writeCaddyfileIfChanged( + "# Generated today\nsite.example.com { respond ok }\n")) + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + after, err := os.Stat(controller.caddyfilePath) + require.NoError(t, err) + assert.True(t, os.SameFile(before, after), "an unchanged body should not replace the file") + }) + + t.Run("preserves the previous file when publication fails", func(t *testing.T) { + controller := testController(t, filepath.Join(t.TempDir(), "missing.sock")) + controller.fileGroup = "uncloud-test-group-that-does-not-exist" + saved := "# previous\nsite.example.com { respond old }\n" + require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640)) + + err := controller.writeCaddyfileIfChanged("# next\nsite.example.com { respond new }\n") + require.ErrorContains(t, err, "change owner of temporary Caddyfile") + got, err := os.ReadFile(controller.caddyfilePath) + require.NoError(t, err) + assert.Equal(t, saved, string(got)) + tmpFiles, err := filepath.Glob(filepath.Join(filepath.Dir(controller.caddyfilePath), ".Caddyfile-*")) + require.NoError(t, err) + assert.Empty(t, tmpFiles) + }) +} + +func TestSelectLocalCaddyContainer(t *testing.T) { + t.Run("prefers a running unhealthy container over a newer stopped one", func(t *testing.T) { + older := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ older }", + "test-machine-id", time.Now().Add(-time.Hour)) + older.Container.State.Health = &container.Health{Status: "unhealthy"} + newer := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ newer }", + "test-machine-id", time.Now()) + newer.Container.State.Running = false + + selected := selectLocalCaddyContainer([]store.ContainerRecord{newer, older}, "test-machine-id") + require.NotNil(t, selected) + assert.Equal(t, older.Container.ID, selected.ID) + }) + + t.Run("ignores remote and hook containers", func(t *testing.T) { + local := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ local }", + "test-machine-id", time.Now().Add(-time.Hour)) + remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ remote }", + "other-machine", time.Now()) + hook := newContainerRecordWithCaddyConfig("caddy", "10.210.0.4", "{ hook }", + "test-machine-id", time.Now()) + hook.Container.Config.Labels[api.LabelHook] = "pre-deploy" + + selected := selectLocalCaddyContainer([]store.ContainerRecord{remote, hook, local}, "test-machine-id") + require.NotNil(t, selected) + assert.Equal(t, local.Container.ID, selected.ID) + }) +} + // TestContainerFingerprint_EqualCoversAllFields is a guard: when a field is added to containerFingerprint, // Equal must also compare it. A mutation of any single field should flip equality to false. If this test fails // after adding a field, update Equal to include it. diff --git a/internal/machine/caddyconfig/jsonconfig.go b/internal/machine/caddyconfig/jsonconfig.go deleted file mode 100644 index 30670bb3..00000000 --- a/internal/machine/caddyconfig/jsonconfig.go +++ /dev/null @@ -1,141 +0,0 @@ -package caddyconfig - -import ( - "encoding/json" - "errors" - "fmt" - "maps" - "net/http" - "slices" - "strconv" - "time" - - "github.com/caddyserver/caddy/v2" - "github.com/caddyserver/caddy/v2/caddyconfig" - "github.com/caddyserver/caddy/v2/modules/caddyhttp" - "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" - "github.com/psviderski/uncloud/pkg/api" -) - -func GenerateJSONConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) { - httpHostUpstreams, httpsHostUpstreams := httpUpstreamsFromPorts(containers) - - var warnings []caddyconfig.Warning - servers := make(map[string]*caddyhttp.Server) - servers["http"] = &caddyhttp.Server{ - Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)}, - // All http requests to this server are logged to the default logger. - Logs: &caddyhttp.ServerLogConfig{}, - Routes: append( - hostUpstreamsToRoutes(httpHostUpstreams, &warnings), - // Add a route to respond with a static verification response at the /.uncloud-verify path. - verificationRoute(verifyResponse, &warnings), - ), - } - servers["https"] = &caddyhttp.Server{ - Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)}, - // All https requests to this server are logged to the default logger. - Logs: &caddyhttp.ServerLogConfig{}, - Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings), - } - - httpApp := caddyhttp.App{ - Servers: servers, - } - config := &caddy.Config{ - AppsRaw: caddy.ModuleMap{ - "http": caddyconfig.JSON(httpApp, &warnings), - }, - } - - var err error - if len(warnings) > 0 { - // warnings only contains errors from JSON marshaling, which are highly unlikely with correct code. - for _, w := range warnings { - err = errors.Join(err, errors.New(w.Message)) - } - return nil, fmt.Errorf("marshal Caddy configuration: %w", err) - } - - return config, nil -} - -// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes. -func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route { - // Sort hostnames for deterministic output. - hostnames := slices.Collect(maps.Keys(hostUpstreams)) - slices.Sort(hostnames) - - routes := make([]caddyhttp.Route, 0, len(hostUpstreams)) - for _, hostname := range hostnames { - upstreams := hostUpstreams[hostname] - upstreamPool := make([]*reverseproxy.Upstream, len(upstreams)) - for i, upstream := range upstreams { - upstreamPool[i] = &reverseproxy.Upstream{ - Dial: upstream, - } - } - handler := &reverseproxy.Handler{ - HealthChecks: &reverseproxy.HealthChecks{ - // Enable passive health checks to automatically detect unhealthy upstreams. - Passive: &reverseproxy.PassiveHealthChecks{ - FailDuration: caddy.Duration(30 * time.Second), - }, - }, - LoadBalancing: &reverseproxy.LoadBalancing{ - // Retry failed requests to skip over temporarily unavailable upstreams. - Retries: 3, - }, - Upstreams: upstreamPool, - } - - routes = append(routes, caddyhttp.Route{ - MatcherSetsRaw: caddyhttp.RawMatcherSets{ - { - "host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings), - }, - }, - HandlersRaw: []json.RawMessage{ - caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings), - }, - }) - } - return routes -} - -// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path. -func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route { - // Return the following route: - // { - // "match": [ - // { - // "path": [ - // "/.uncloud-verify" - // ] - // } - // ], - // "handle": [ - // { - // "handler": "static_response", - // "body": "", - // "status_code": 200 - // } - // ] - // } - - staticResponse := caddyhttp.StaticResponse{ - StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)), - Body: response, - } - - return caddyhttp.Route{ - MatcherSetsRaw: caddyhttp.RawMatcherSets{ - { - "path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings), - }, - }, - HandlersRaw: []json.RawMessage{ - caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings), - }, - } -} diff --git a/internal/machine/caddyconfig/jsonconfig_test.go b/internal/machine/caddyconfig/jsonconfig_test.go deleted file mode 100644 index 77f95bd6..00000000 --- a/internal/machine/caddyconfig/jsonconfig_test.go +++ /dev/null @@ -1,378 +0,0 @@ -package caddyconfig - -import ( - "strings" - "testing" - - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" - "github.com/psviderski/uncloud/internal/machine/docker" - "github.com/psviderski/uncloud/pkg/api" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGenerateJSONConfig(t *testing.T) { - configWithoutServices := `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [{ - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - }], - "logs": {} - }, - "https": { - "listen": [":443"], - "logs": {} - } - } - }` - - tests := []struct { - name string - containers []api.ServiceContainer - want string - wantErr bool - }{ - { - name: "empty containers", - containers: []api.ServiceContainer{}, - want: configWithoutServices, - wantErr: false, - }, - - { - name: "HTTP container", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), - }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [ - { - "match": [{"host": ["app.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [{"dial": "10.210.0.2:8080"}] - }] - }, - { - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - } - ], - "logs": {} - }, - "https": { - "listen": [":443"], - "logs": {} - } - } - }`, - wantErr: false, - }, - { - name: "load balancing multiple containers", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "app.example.com:8080/http"), - newContainer("10.210.0.3", "app.example.com:8080/http"), - }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [ - { - "match": [{"host": ["app.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [ - {"dial": "10.210.0.2:8080"}, - {"dial": "10.210.0.3:8080"} - ] - }] - }, - { - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - } - ], - "logs": {} - }, - "https": { - "listen": [":443"], - "logs": {} - } - } - }`, - wantErr: false, - }, - { - name: "HTTPS container", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "secure.example.com:8000/https"), - }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [ - { - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - } - ], - "logs": {} - }, - "https": { - "listen": [":443"], - "routes": [ - { - "match": [{"host": ["secure.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [{"dial": "10.210.0.2:8000"}] - }] - } - ], - "logs": {} - } - } - }`, - wantErr: false, - }, - { - name: "mixed HTTP and HTTPS", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", - "app.example.com:8080/http", - "web.example.com:8000/http"), - newContainer("10.210.0.3", - "app.example.com:8080/http", - "secure.example.com:8888/https"), - newContainer("10.210.0.4", - "web.example.com:8000/http", - "secure.example.com:8888/https"), - newContainer("10.210.0.5", - "app.example.com:8080/http", - "web.example.com:8000/http", - "secure.example.com:8888/https"), - }, - want: `{ - "servers": { - "http": { - "listen": [":80"], - "routes": [ - { - "match": [{"host": ["app.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [ - {"dial": "10.210.0.2:8080"}, - {"dial": "10.210.0.3:8080"}, - {"dial": "10.210.0.5:8080"} - ] - }] - }, - { - "match": [{"host": ["web.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [ - {"dial": "10.210.0.2:8000"}, - {"dial": "10.210.0.4:8000"}, - {"dial": "10.210.0.5:8000"} - ] - }] - }, - { - "match": [{"path": ["/.uncloud-verify"]}], - "handle": [{ - "body": "verification-response-body", - "handler": "static_response", - "status_code": 200 - }] - } - ], - "logs": {} - }, - "https": { - "listen": [":443"], - "routes": [ - { - "match": [{"host": ["secure.example.com"]}], - "handle": [{ - "handler": "reverse_proxy", - "health_checks": { - "passive": { - "fail_duration": 30000000000 - } - }, - "load_balancing": { - "retries": 3 - }, - "upstreams": [ - {"dial": "10.210.0.3:8888"}, - {"dial": "10.210.0.4:8888"}, - {"dial": "10.210.0.5:8888"} - ] - }] - } - ], - "logs": {} - } - } - }`, - wantErr: false, - }, - { - name: "container without uncloud network ignored", - containers: []api.ServiceContainer{ - newContainerWithoutNetwork("ignored.example.com:8080/http"), - }, - want: configWithoutServices, - wantErr: false, - }, - { - name: "container with invalid port ignored", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "invalid-port"), - }, - want: configWithoutServices, - wantErr: false, - }, - { - name: "containers with unsupported protocols and host mode ignored", - containers: []api.ServiceContainer{ - newContainer("10.210.0.2", "5000/tcp"), - newContainer("10.210.0.3", "5000/udp"), - newContainer("10.210.0.4", "80:8080/tcp@host"), - }, - want: configWithoutServices, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config, err := GenerateJSONConfig(tt.containers, "verification-response-body") - - if tt.wantErr { - assert.Error(t, err) - return - } - require.NoError(t, err) - - require.Len(t, config.AppsRaw, 1, "Expected one http app") - require.Contains(t, config.AppsRaw, "http", "Expected http app") - - assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match") - }) - } -} - -func newContainer(ip string, ports ...string) api.ServiceContainer { - portsLabel := strings.Join(ports, ",") - return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{ - ContainerJSONBase: &container.ContainerJSONBase{ - State: &container.State{ - Running: true, - }, - }, - NetworkSettings: &container.NetworkSettings{ - Networks: map[string]*network.EndpointSettings{ - docker.NetworkName: { - IPAddress: ip, - }, - }, - }, - Config: &container.Config{ - Labels: map[string]string{ - api.LabelServicePorts: portsLabel, - }, - }, - }}} -} - -func newContainerWithoutNetwork(ports ...string) api.ServiceContainer { - portsLabel := strings.Join(ports, ",") - return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{ - ContainerJSONBase: &container.ContainerJSONBase{ - State: &container.State{ - Running: true, - }, - }, - NetworkSettings: &container.NetworkSettings{ - Networks: map[string]*network.EndpointSettings{ - "other-network": { - IPAddress: "172.17.0.2", - }, - }, - }, - Config: &container.Config{ - Labels: map[string]string{ - api.LabelServicePorts: portsLabel, - }, - }, - }}} -} diff --git a/internal/machine/caddyconfig/server.go b/internal/machine/caddyconfig/server.go index 22c1d6de..24ea9c4e 100644 --- a/internal/machine/caddyconfig/server.go +++ b/internal/machine/caddyconfig/server.go @@ -2,6 +2,7 @@ package caddyconfig import ( "context" + "errors" "os" "google.golang.org/grpc/codes" @@ -22,18 +23,26 @@ func NewServer(service *Service) *Server { return &Server{service: service} } -// GetConfig retrieves the current Caddy configuration from the machine. +// GetConfig retrieves the saved Caddy configuration and the latest reconciliation error from the machine. func (s *Server) GetConfig(ctx context.Context, _ *emptypb.Empty) (*pb.GetCaddyConfigResponse, error) { caddyfile, modifiedAt, err := s.service.Caddyfile() if err != nil { - if os.IsNotExist(err) { + if errors.Is(err, os.ErrNotExist) { + if lastErr := s.service.LastReconciliationError(); lastErr != nil { + return nil, status.Errorf(codes.NotFound, "%v; last Caddy config load failed: %v", err, lastErr) + } return nil, status.Error(codes.NotFound, err.Error()) } return nil, status.Error(codes.Internal, err.Error()) } + recErr := "" + if lastErr := s.service.LastReconciliationError(); lastErr != nil { + recErr = lastErr.Error() + } return &pb.GetCaddyConfigResponse{ - Caddyfile: caddyfile, - ModifiedAt: timestamppb.New(modifiedAt), + Caddyfile: caddyfile, + ModifiedAt: timestamppb.New(modifiedAt), + LastReconciliationError: recErr, }, nil } diff --git a/internal/machine/caddyconfig/server_test.go b/internal/machine/caddyconfig/server_test.go new file mode 100644 index 00000000..944a2fbe --- /dev/null +++ b/internal/machine/caddyconfig/server_test.go @@ -0,0 +1,62 @@ +package caddyconfig + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +func TestServer_GetConfig(t *testing.T) { + t.Run("returns saved Caddyfile and reconciliation error", func(t *testing.T) { + dir := t.TempDir() + caddyfile := "example.com { respond ok }\n" + path := filepath.Join(dir, "Caddyfile") + require.NoError(t, os.WriteFile(path, []byte(caddyfile), 0o640)) + info, err := os.Stat(path) + require.NoError(t, err) + + service := NewService(dir) + service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud")) + config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{}) + + require.NoError(t, err) + assert.Equal(t, caddyfile, config.GetCaddyfile()) + assert.True(t, info.ModTime().Equal(config.GetModifiedAt().AsTime())) + assert.Equal(t, "module not registered: caddy.storage.uncloud", config.GetLastReconciliationError()) + + service.setReconciliationResult(nil) + config, err = NewServer(service).GetConfig(context.Background(), &emptypb.Empty{}) + require.NoError(t, err) + assert.Empty(t, config.GetLastReconciliationError()) + }) + + t.Run("returns NotFound with reconciliation error when Caddyfile is absent", func(t *testing.T) { + service := NewService(t.TempDir()) + service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud")) + + config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{}) + + assert.Nil(t, config) + assert.Equal(t, codes.NotFound, status.Code(err)) + assert.ErrorContains(t, err, "Caddyfile") + assert.ErrorContains(t, err, "last Caddy config load failed: module not registered: caddy.storage.uncloud") + }) + + t.Run("returns NotFound without reconciliation error when Caddyfile is absent", func(t *testing.T) { + service := NewService(t.TempDir()) + + config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{}) + + assert.Nil(t, config) + assert.Equal(t, codes.NotFound, status.Code(err)) + assert.NotContains(t, err.Error(), "last Caddy config load failed") + }) +} diff --git a/internal/machine/caddyconfig/service.go b/internal/machine/caddyconfig/service.go index 4ba72ba9..ca8483ce 100644 --- a/internal/machine/caddyconfig/service.go +++ b/internal/machine/caddyconfig/service.go @@ -4,12 +4,15 @@ import ( "fmt" "os" "path/filepath" + "sync" "time" ) // Service provides methods to interact with the Caddy configuration on the machine. type Service struct { - configDir string + configDir string + mu sync.RWMutex + lastReconciliationError error } // NewService creates a new Service instance with the specified Caddy configuration directory. @@ -17,7 +20,8 @@ func NewService(configDir string) *Service { return &Service{configDir: configDir} } -// Caddyfile retrieves the current Caddy configuration (Caddyfile) from the machine's config directory. +// Caddyfile retrieves the saved Caddyfile from the machine's config directory. The saved file may differ from the +// running configuration if Caddy accepted a load but the subsequent write failed. func (s *Service) Caddyfile() (string, time.Time, error) { path := filepath.Join(s.configDir, "Caddyfile") content, err := os.ReadFile(path) @@ -33,3 +37,17 @@ func (s *Service) Caddyfile() (string, time.Time, error) { return string(content), fileInfo.ModTime(), nil } + +// LastReconciliationError reports the most recent unsuccessful controller attempt, if any. A successful attempt +// clears it. An empty value does not prove that Caddy has loaded the saved Caddyfile. +func (s *Service) LastReconciliationError() error { + s.mu.RLock() + defer s.mu.RUnlock() + return s.lastReconciliationError +} + +func (s *Service) setReconciliationResult(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.lastReconciliationError = err +} diff --git a/internal/machine/caddyconfig/service_test.go b/internal/machine/caddyconfig/service_test.go new file mode 100644 index 00000000..6d56a516 --- /dev/null +++ b/internal/machine/caddyconfig/service_test.go @@ -0,0 +1,42 @@ +package caddyconfig + +import ( + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestService_LastReconciliationError(t *testing.T) { + service := NewService(t.TempDir()) + assert.NoError(t, service.LastReconciliationError()) + + service.setReconciliationResult(errors.New("global Caddy config rejected")) + assert.EqualError(t, service.LastReconciliationError(), "global Caddy config rejected") + + service.setReconciliationResult(nil) + assert.NoError(t, service.LastReconciliationError()) +} + +func TestService_LastReconciliationErrorConcurrent(t *testing.T) { + service := NewService(t.TempDir()) + var wg sync.WaitGroup + for range 10 { + wg.Add(2) + go func() { + defer wg.Done() + for range 100 { + service.setReconciliationResult(errors.New("retry")) + service.setReconciliationResult(nil) + } + }() + go func() { + defer wg.Done() + for range 100 { + _ = service.LastReconciliationError() + } + }() + } + wg.Wait() +} diff --git a/internal/machine/machine.go b/internal/machine/machine.go index c678cf61..3f6f5b06 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -198,6 +198,8 @@ type Machine struct { dockerServer *machinedocker.Server // machineAPIServer handles API requests directly on this machine. machineAPIServer *grpc.Server + // caddyService provides methods to interact with the Caddy configuration on the machine. + caddyService *caddyconfig.Service // proxyDirector routes API requests to local or remote machines. proxyDirector *apiproxy.Director @@ -302,6 +304,7 @@ func NewMachine(config *Config) (*Machine, error) { dockerService: dockerService, clusterAPIServer: clusterAPIServer, proxyDirector: proxyDirector, + caddyService: caddyconfig.NewService(config.CaddyConfigDir), } // Machine IP will only be available after the machine is initialised as a cluster member so wrap it in a function. @@ -316,7 +319,7 @@ func NewMachine(config *Config) (*Machine, error) { NetworkReady: m.IsNetworkReady, WaitForNetworkReady: m.WaitForNetworkReady, }) - caddyServer := caddyconfig.NewServer(caddyconfig.NewService(config.CaddyConfigDir)) + caddyServer := caddyconfig.NewServer(m.caddyService) caddyStore, err := corroStore.Keyspace(caddystorage.Namespace) if err != nil { @@ -535,7 +538,7 @@ func (m *Machine) Run(ctx context.Context) error { // It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability. caddyconfigCtrl, err := caddyconfig.NewController( m.state.ID, - m.config.CaddyConfigDir, + m.caddyService, DefaultCaddyAdminSockPath, m.store, ) diff --git a/test/e2e/machine_test.go b/test/e2e/machine_test.go index f3a4a709..a70765d7 100644 --- a/test/e2e/machine_test.go +++ b/test/e2e/machine_test.go @@ -10,6 +10,7 @@ import ( "github.com/psviderski/uncloud/api/pb" "github.com/psviderski/uncloud/internal/ucind" "github.com/psviderski/uncloud/pkg/api" + "github.com/psviderski/uncloud/pkg/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -456,6 +457,23 @@ func TestMachineOperations(t *testing.T) { }) t.Run("remove machine clears container records from cluster store", func(t *testing.T) { + // The Caddy controller needs a local Caddy container to supply the global config before it can generate + // routes. Place it on the connected machine by ID because earlier tests rename that machine. + caddyDeployment, err := cli.NewCaddyDeployment("", "", api.Placement{ + Machines: []string{c.Machines[0].ID}, + }) + require.NoError(t, err) + monitorPeriod := 5 * time.Second + caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &monitorPeriod + _, err = caddyDeployment.Run(ctx) + require.NoError(t, err) + t.Cleanup(func() { + err := cli.RemoveService(ctx, client.CaddyServiceName) + if err != nil && !errors.Is(err, api.ErrNotFound) { + assert.NoError(t, err) + } + }) + // Deploy a global service with an HTTP ingress port so the auto-generated Caddyfile lists // each container's IP as an upstream. serviceName := "test-machine-rm-cleanup" diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 733f390f..03b07647 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -27,6 +27,8 @@ import ( "github.com/psviderski/uncloud/pkg/client/deploy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func newServiceID() string { @@ -42,6 +44,9 @@ func TestDeployment(t *testing.T) { clusterName := "ucind-test.deployment" ctx := context.Background() + // Caddy may restart once while the controller writes its bootstrap file. Use the normal deployment monitor period + // instead of the zero-duration override used by most e2e tests. + caddyMonitorPeriod := 5 * time.Second c, _ := createTestCluster(t, clusterName, ucind.CreateClusterOptions{Machines: 3}, true) cli, cErr := c.Machines[0].Connect(ctx) @@ -313,6 +318,7 @@ func TestDeployment(t *testing.T) { deployment, err := cli.NewCaddyDeployment("", "", api.Placement{}) require.NoError(t, err) + deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod _, err = deployment.Run(ctx) require.NoError(t, err) @@ -345,6 +351,7 @@ func TestDeployment(t *testing.T) { Machines: []string{c.Machines[0].Name}, }) require.NoError(t, err) + deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod image := deployment.Spec.Container.Image _, err = deployment.Run(ctx) @@ -361,6 +368,7 @@ func TestDeployment(t *testing.T) { // Deploy to all machines without a placement constraint. deployment, err = cli.NewCaddyDeployment(image, "", api.Placement{}) require.NoError(t, err) + deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod _, err = deployment.Run(ctx) require.NoError(t, err) @@ -391,6 +399,13 @@ func TestDeployment(t *testing.T) { } }) + // Without a Caddy container, the controller has no global config to pair with application configs. Keep any + // previously saved Caddyfile unchanged rather than publishing a new one without Caddy's global settings. + savedBefore, savedBeforeErr := cli.Caddy.GetConfig(ctx, nil) + if savedBeforeErr != nil { + require.Equal(t, codes.NotFound, status.Code(savedBeforeErr)) + } + // First deploy a service with custom caddy config before caddy is deployed. serviceCaddyfile := `test-custom-caddy-config.example.com { reverse_proxy {{upstreams}} { @@ -416,17 +431,13 @@ func TestDeployment(t *testing.T) { require.NoError(t, err) assertServiceMatchesSpec(t, svc, spec) - // Check that the generated Caddyfile contains a comment that user-define configs were skipped. - var config *pb.GetCaddyConfigResponse - require.Eventually(t, func() bool { - config, err = cli.Caddy.GetConfig(ctx, nil) - if err != nil { - return false + require.Never(t, func() bool { + current, currentErr := cli.Caddy.GetConfig(ctx, nil) + if savedBeforeErr != nil { + return currentErr == nil } - return strings.Contains(config.Caddyfile, "# NOTE: User-defined configs for services were skipped") - }, 5*time.Second, 100*time.Millisecond) - - assert.NotContains(t, config.Caddyfile, "test-custom-caddy-config.example.com {") + return currentErr == nil && current.Caddyfile != savedBefore.Caddyfile + }, 2*time.Second, 100*time.Millisecond) // Now deploy caddy with custom config. caddyCaddyfile := `{ @@ -438,6 +449,7 @@ myapp.example.com { }` caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{}) require.NoError(t, err) + caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod _, err = caddyDeployment.Run(ctx) require.NoError(t, err) @@ -447,6 +459,7 @@ myapp.example.com { assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec) // Wait for the Caddyfile to be regenerated with both custom configs. + var config *pb.GetCaddyConfigResponse require.Eventually(t, func() bool { config, err = cli.Caddy.GetConfig(ctx, nil) if err != nil {