PhilSchlo/uncloud
1
0 Fork 0
mirror of https://github.com/psviderski/uncloud.git synced 2026-10-06 13:18:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(caddyconfig): custom global Caddy config is preserved on regeneration,warn about last load error in 'uc caddy config' (fixes #412)

Browse Source
This commit is contained in:
Pasha Sviderski committed 2026-09-24 22:24:42 +10:00
1 parent 45d33d87dd
commit ace8cbf974
18 files changed
+1200 -837
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

api/pb/caddy.pb.go
+30 -17
View File
@@ -27,10 +27,12 @@ type GetCaddyConfigResponse struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The generated Caddyfile content.
// The saved Caddyfile content.
Caddyfile string `protobuf:"bytes,1,opt,name=caddyfile,proto3" json:"caddyfile,omitempty"`
// Timestamp when the config was last modified.
ModifiedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=modified_at,json=modifiedAt,proto3" json:"modified_at,omitempty"`
// Error from the latest unsuccessful reconciliation attempt.
LastReconciliationError string `protobuf:"bytes,3,opt,name=last_reconciliation_error,json=lastReconciliationError,proto3" json:"last_reconciliation_error,omitempty"`
}
func (x *GetCaddyConfigResponse) Reset() {
@@ -79,6 +81,13 @@ func (x *GetCaddyConfigResponse) GetModifiedAt() *timestamppb.Timestamp {
return nil
}
func (x *GetCaddyConfigResponse) GetLastReconciliationError() string {
if x != nil {
return x.LastReconciliationError
}
return ""
}
var File_api_pb_caddy_proto protoreflect.FileDescriptor
var file_api_pb_caddy_proto_rawDesc = []byte{
@@ -87,22 +96,26 @@ var file_api_pb_caddy_proto_rawDesc = []byte{
0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x61,
0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x12,
0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x32, 0x49, 0x0a, 0x05,
0x43, 0x61, 0x64, 0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69,
0x2e, 0x47, 0x65, 0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69,
0x2f, 0x75, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xaf, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43,
0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x64, 0x64, 0x79, 0x66, 0x69, 0x6c, 0x65,
0x12, 0x3b, 0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18,
0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
0x70, 0x52, 0x0a, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, 0x0a,
0x19, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x17, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x63, 0x69, 0x6c, 0x69, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x32, 0x49, 0x0a, 0x05, 0x43, 0x61, 0x64,
0x64, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65,
0x74, 0x43, 0x61, 0x64, 0x64, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x70, 0x73, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x6b, 0x69, 0x2f, 0x75, 0x6e,
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (
api/pb/caddy.proto
+3 -1
View File
@@ -13,8 +13,10 @@ service Caddy {
}
message GetCaddyConfigResponse {
// The generated Caddyfile content.
// The saved Caddyfile content.
string caddyfile = 1;
// Timestamp when the config was last modified.
google.protobuf.Timestamp modified_at = 2;
// Error from the latest unsuccessful reconciliation attempt.
string last_reconciliation_error = 3;
}
cmd/uc/caddy/config.go
+6
View File
@@ -8,6 +8,7 @@ import (
"github.com/alecthomas/chroma/v2/quick"
"github.com/psviderski/uncloud/internal/cli"
"github.com/psviderski/uncloud/internal/cli/completion"
"github.com/psviderski/uncloud/internal/cli/tui"
"github.com/spf13/cobra"
)
@@ -66,5 +67,10 @@ func runConfig(ctx context.Context, uncli *cli.CLI, opts configOptions) error {
}
}
if config.LastReconciliationError != "" {
tui.PrintWarning(fmt.Sprintf("last Caddy config load failed: %s\nShowing the last saved Caddyfile.",
config.LastReconciliationError))
}
return nil
}
internal/machine/caddyconfig/caddyfile.go
+57 -55
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"log/slog"
"maps"
@@ -57,14 +58,12 @@ https://{{$hostname}} {
log
}{{end}}
`
caddyfileUnavailabeFooter = `# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
# and its logs for more details (uc logs caddy).
`
bootstrapMarker = "# Uncloud bootstrap for Caddy container "
legacyBootstrapMarker = "# NOTE: User-defined configs for services were skipped because Caddy is not running"
)
// CaddyfileGenerator generates a Caddyfile configuration for the Caddy reverse proxy.
// It combines generated routes from published ports with user-defined Caddyfile snippets from service specs (x-caddy).
type CaddyfileGenerator struct {
// machineID is the unique identifier of the machine where the controller is running.
machineID string
@@ -74,11 +73,18 @@ type CaddyfileGenerator struct {
log *slog.Logger
}
// CaddyfileValidator is an interface for validating Caddyfile configurations.
// CaddyfileValidator checks a candidate Caddyfile without loading it. Validate returns InvalidCaddyfileError only
// when the candidate is known to be invalid. Other errors mean the check could not be completed. A successful check
// does not guarantee that the configuration will load.
type CaddyfileValidator interface {
Validate(ctx context.Context, caddyfile string) error
}
// InvalidCaddyfileError means validation completed and the candidate Caddyfile was rejected.
type InvalidCaddyfileError struct{ Message string }
func (e *InvalidCaddyfileError) Error() string { return e.Message }
func NewCaddyfileGenerator(
machineID, machineName string, validator CaddyfileValidator, log *slog.Logger,
) *CaddyfileGenerator {
@@ -93,35 +99,41 @@ func NewCaddyfileGenerator(
}
}
// Generate creates a Caddyfile configuration based on the provided service containers.
// The Caddyfile is generated from the service ports of the healthy containers.
// If a 'caddy' service container is running on this machine and defines a custom Caddy config (x-caddy) in its service
// spec, it will be validated and prepended to the generated Caddyfile. Custom Caddy configs (x-caddy) defined in other
// service specs are validated and appended to the generated Caddyfile. Invalid configs are logged and skipped to ensure
// the generated Caddyfile remains valid.
// Generate creates a Caddyfile for the local Caddy container from the supplied healthy application containers.
// Application service ports provide the generated sites. The Caddy container's custom Caddy config (x-caddy), if
// defined, provides the global block even when that container is unhealthy. When application custom Caddy configs
// are included, the newest container for each service provides its config.
//
// The final Caddyfile structure includes:
// The resulting Caddyfile has this structure:
//
// [caddy x-caddy (global config)]
// [generated Caddyfile from all service ports]
// [service-a x-caddy]
// [caddy custom Caddy config (global)]
// [generated sites from application service ports]
// [service-a custom Caddy config]
// ...
// [service-z x-caddy]
// [service-z custom Caddy config]
//
// If includeCustom is false, custom Caddy configs (x-caddy) are not included in the generated Caddyfile.
// Bootstrap mode keeps the global custom Caddy config and generated sites, but omits application custom Caddy configs.
// It skips validation because Caddy may not be running yet. In this case, Caddy validates the config when it starts.
// Global template errors still stop generation. In full mode, invalid globals stop generation, while invalid
// application custom Caddy configs are logged and skipped.
func (g *CaddyfileGenerator) Generate(
ctx context.Context, records []store.ContainerRecord, includeCustom bool,
ctx context.Context,
caddyCtr api.ServiceContainer,
records []store.ContainerRecord,
bootstrap bool,
) (string, error) {
records = slices.Clone(records)
// Sort records by local machine first, then by service name and creation time. Placing containers on the local
// machine first lets user-defined Caddy configs pair this ordering with the "first" lb_policy to always send
// traffic to the same-host replica (skipping the cross-machine hop) and only fall back to remote upstreams when
// the local one is unhealthy.
// The service name and creation time tiebreakers keep the generated Caddyfile stable across regenerations.
// The service name, creation time, and container ID tiebreakers keep the file stable across regenerations.
slices.SortStableFunc(records, func(a, b store.ContainerRecord) int {
return cmp.Or(
g.localMachineRank(a.MachineID)-g.localMachineRank(b.MachineID),
strings.Compare(a.Container.ServiceName(), b.Container.ServiceName()),
a.Container.CreatedTime().Compare(b.Container.CreatedTime()),
strings.Compare(a.Container.ID, b.Container.ID),
)
})
@@ -136,26 +148,11 @@ func (g *CaddyfileGenerator) Generate(
}
caddyfileHeader := fmt.Sprintf(caddyfileHeaderFmt, g.machineName, time.Now().UTC().Format(time.RFC3339))
if !includeCustom {
return fmt.Sprintf("%s\n%s\n%s", caddyfileHeader, caddyfile, caddyfileUnavailabeFooter), nil
}
upstreams := serviceUpstreams(containers)
// Track validation errors for reporting.
var configErrors []string
// Find the 'caddy' service container on this machine. Use the most recent one if multiple exist.
var caddyCtr *api.ServiceContainer
for _, cr := range records {
if cr.MachineID == g.machineID && cr.Container.ServiceName() == CaddyServiceName &&
(caddyCtr == nil || cr.Container.CreatedTime().Compare(caddyCtr.CreatedTime()) > 0) {
caddyCtr = &cr.Container
}
}
// If the caddy container is running on this machine and has a custom Caddy config (global),
// prepend it to the generated Caddyfile and validate it.
if caddyCtr != nil && caddyCtr.ServiceSpec.CaddyConfig() != "" {
if caddyCtr.ServiceSpec.CaddyConfig() != "" {
// Render the custom global Caddy config as a Go template with the upstreams.
tmplCtx := templateContext{
Name: caddyCtr.ServiceName(),
@@ -163,23 +160,23 @@ func (g *CaddyfileGenerator) Generate(
}
renderedConfig, err := renderCaddyfile(tmplCtx, caddyCtr.ServiceSpec.CaddyConfig())
if err != nil {
g.log.Error("Failed to render template directives in user-defined global Caddy config, skipping it.",
"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
configErrors = append(configErrors,
fmt.Sprintf("service '%s': failed to render template: %v", caddyCtr.ServiceName(), err))
} else {
return "", fmt.Errorf(
"render template directives in user-defined global Caddy config from Caddy container %s: %w",
caddyCtr.ShortID(), err)
}
caddyfileCandidate := fmt.Sprintf("# User-defined global config from service '%s'.\n%s\n\n%s",
caddyCtr.ServiceName(), renderedConfig, caddyfile)
if !bootstrap && g.validator != nil {
if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
g.log.Error("User-defined global Caddy config is invalid, skipping it.",
"service", caddyCtr.ServiceName(), "container", caddyCtr.ID, "err", err)
configErrors = append(configErrors,
fmt.Sprintf("service '%s': validation failed: %v", caddyCtr.ServiceName(), err))
} else {
return "", fmt.Errorf("validate user-defined global Caddy config from Caddy container %s: %w",
caddyCtr.ShortID(), err)
}
}
caddyfile = caddyfileCandidate
}
}
if bootstrap {
return caddyfileHeader + bootstrapMarker + caddyCtr.ID + "\n\n" + caddyfile, nil
}
// There could be multiple service containers for the same service with different custom Caddy configs, for example,
@@ -200,7 +197,7 @@ func (g *CaddyfileGenerator) Generate(
// Append a custom Caddy config for each service to the Caddyfile and validate it. If the config for a service
// is invalid, skip it but continue processing other services to ensure the Caddyfile remains valid.
for _, serviceName := range sortedServiceNames {
// Skip the caddy container as we already processed it.
// Skip the caddy container as we already processed it as the global config.
if serviceName == CaddyServiceName {
continue
}
@@ -226,16 +223,22 @@ func (g *CaddyfileGenerator) Generate(
caddyfileCandidate := fmt.Sprintf("%s\n# User-defined config for service '%s'.\n%s\n",
caddyfile, serviceName, renderedConfig)
if g.validator != nil {
if err = g.validator.Validate(ctx, caddyfileCandidate); err != nil {
if _, ok := errors.AsType[*InvalidCaddyfileError](err); !ok {
return "", fmt.Errorf("validate Caddy config for service '%s': %w", serviceName, err)
}
g.log.Error("User-defined Caddy config for service is invalid, skipping it.",
"service", serviceName, "err", err)
configErrors = append(configErrors, fmt.Sprintf("service '%s': validation failed: %v", serviceName, err))
} else {
configErrors = append(configErrors,
fmt.Sprintf("service '%s': validation failed: %v", serviceName, err))
continue
}
}
caddyfile = caddyfileCandidate
}
}
// Append error summary as comment if there were any invalid configs.
// Keep skipped-snippet errors in the saved file so an operator can see why routes are missing after a restart.
if len(configErrors) > 0 {
var errorsComment strings.Builder
errorsComment.WriteString("# Skipped invalid user-defined configs:\n")
@@ -249,8 +252,7 @@ func (g *CaddyfileGenerator) Generate(
return caddyfileHeader + "\n" + caddyfile, nil
}
// localMachineRank returns 0 if the given machineID matches the local machine and 1 otherwise.
// Useful for sorting containers running locally first.
// localMachineRank is a sorting function for containers that puts running on the local machine first.
func (g *CaddyfileGenerator) localMachineRank(machineID string) int {
if g.machineID == machineID {
return 0
internal/machine/caddyconfig/caddyfile_test.go
+187 -116
View File
@@ -208,10 +208,11 @@ http://app.example.com {
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Validator is not expected to be called in these tests.
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil)
validator := NewMockCaddyfileValidator(t)
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
config, err := generator.Generate(ctx, tt.containers, true)
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.1", "", "", time.Now()).Container
config, err := generator.Generate(ctx, caddyCtr, tt.containers, false)
if tt.wantErr {
assert.Error(t, err)
@@ -332,7 +333,7 @@ bad.template.com {
`,
},
{
name: "caddy service with invalid global config is skipped",
name: "caddy service with invalid global config aborts",
containers: []store.ContainerRecord{
newContainerRecordWithCaddyConfig(
"caddy",
@@ -345,10 +346,7 @@ localhost {
time.Now(),
),
},
want: testCaddyfileHeader + `
# Skipped invalid user-defined configs:
# - service 'caddy': validation failed: invalid config detected
`,
wantErr: true,
},
{
name: "caddy service on different machine is ignored",
@@ -786,7 +784,7 @@ web-v2.example.com {
`,
},
{
name: "multiple errors: invalid global, template error, and validation error",
name: "invalid global aborts before application configs",
containers: []store.ContainerRecord{
newContainerRecordWithCaddyConfig(
"caddy",
@@ -827,17 +825,7 @@ invalid.example.com {
time.Now(),
),
},
want: testCaddyfileHeader + `
# User-defined config for service 'valid'.
valid.example.com {
respond "Valid config"
}
# Skipped invalid user-defined configs:
# - service 'caddy': validation failed: invalid config detected
# - service 'broken-template': failed to render template: parse config as Go template: template: Caddyfile:2: unterminated quoted string
# - service 'invalid': validation failed: invalid config detected
`,
wantErr: true,
},
}
@@ -846,7 +834,7 @@ valid.example.com {
validator.EXPECT().Validate(mock.Anything, mock.Anything).RunAndReturn(
func(ctx context.Context, caddyfile string) error {
if strings.Contains(caddyfile, "# test:invalid") {
return errors.New("invalid config detected")
return &InvalidCaddyfileError{Message: "invalid config detected"}
}
return nil
})
@@ -855,7 +843,8 @@ valid.example.com {
t.Run(tt.name, func(t *testing.T) {
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
config, err := generator.Generate(ctx, tt.containers, true)
caddyCtr := caddyContainerFromTestContainers(tt.containers)
config, err := generator.Generate(ctx, caddyCtr, tt.containers, false)
if tt.wantErr {
assert.Error(t, err)
@@ -868,6 +857,167 @@ valid.example.com {
}
}
func TestCaddyfileGeneratorBootstrap(t *testing.T) {
// Bootstrap keeps the local Caddy container's globals and generated routes, but omits application custom configs.
tests := []struct {
name string
containers []store.ContainerRecord
want string
}{
{
name: "global config retained and application custom configs skipped",
containers: []store.ContainerRecord{
newContainerRecordWithCaddyConfig(
"caddy",
"10.210.0.1",
`# Global config
{
global directive
}`,
"test-machine-id",
time.Now(),
),
newContainerRecordWithCaddyConfig(
"web",
"10.210.0.2",
`web.example.com {
reverse_proxy web:3000
}`,
"test-machine-id",
time.Now(),
),
newContainerRecordWithPorts(
"api",
"10.210.0.3",
[]string{"api.example.com:8080/http"},
"test-machine-id",
),
},
want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", `
# Uncloud bootstrap for Caddy container caddy-10.210.0.1
# User-defined global config from service 'caddy'.
# Global config
{
global directive
}
# Health check endpoint`, 1) + `
# Sites generated from service ports.
http://api.example.com {
reverse_proxy 10.210.0.3:8080 {
import common_proxy
}
log
}
`,
},
{
name: "no containers with x-caddy configs",
containers: []store.ContainerRecord{
newContainerRecordWithPorts(
"api",
"10.210.0.3",
[]string{"api.example.com:8080/http"},
"test-machine-id",
),
},
want: strings.Replace(testCaddyfileHeader, "\n\n# Health check endpoint", `
# Uncloud bootstrap for Caddy container caddy-10.210.0.1
# Health check endpoint`, 1) + `
# Sites generated from service ports.
http://api.example.com {
reverse_proxy 10.210.0.3:8080 {
import common_proxy
}
log
}
`,
},
}
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
validator := NewMockCaddyfileValidator(t)
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
caddyContainer := caddyContainerFromTestContainers(tt.containers)
config, err := generator.Generate(ctx, caddyContainer, tt.containers, true)
require.NoError(t, err)
assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match")
})
}
}
func TestCaddyfileGenerator_ValidatorTransportErrorAbortsGeneration(t *testing.T) {
validator := NewMockCaddyfileValidator(t)
validator.EXPECT().Validate(mock.Anything, mock.Anything).Return(errors.New("socket disappeared"))
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", validator, nil)
app := newContainerRecordWithCaddyConfig(
"web",
"10.210.0.2",
"web.example.com { respond ok }",
"test-machine-id",
time.Now(),
)
_, err := generator.Generate(context.Background(), caddyContainerFromTestContainers(nil),
[]store.ContainerRecord{app}, false)
require.ErrorContains(t, err, "socket disappeared")
}
func newContainer(ip string, ports ...string) api.ServiceContainer {
portsLabel := strings.Join(ports, ",")
return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{
State: &container.State{
Running: true,
},
},
NetworkSettings: &container.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
docker.NetworkName: {
IPAddress: ip,
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
}}}
}
func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
portsLabel := strings.Join(ports, ",")
return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{
State: &container.State{
Running: true,
},
},
NetworkSettings: &container.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
"other-network": {
IPAddress: "172.17.0.2",
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
}}}
}
func newContainerRecord(ctr api.ServiceContainer, machineID string) store.ContainerRecord {
return store.ContainerRecord{
Container: ctr,
@@ -911,100 +1061,6 @@ func newContainerRecordWithCaddyConfig(serviceName, ip, caddyConfig, machineID s
}
}
func TestCaddyfileGeneratorWithoutCustomConfigs(t *testing.T) {
// Test that when includeCustom is false (Caddy not available), x-caddy configs are skipped.
tests := []struct {
name string
containers []store.ContainerRecord
want string
}{
{
name: "x-caddy configs are skipped",
containers: []store.ContainerRecord{
newContainerRecordWithCaddyConfig(
"caddy",
"10.210.0.1",
`# Global config
{
global directive
}`,
"test-machine-id",
time.Now(),
),
newContainerRecordWithCaddyConfig(
"web",
"10.210.0.2",
`web.example.com {
reverse_proxy web:3000
}`,
"test-machine-id",
time.Now(),
),
newContainerRecordWithPorts(
"api",
"10.210.0.3",
[]string{"api.example.com:8080/http"},
"test-machine-id",
),
},
want: testCaddyfileHeader + `
# Sites generated from service ports.
http://api.example.com {
reverse_proxy 10.210.0.3:8080 {
import common_proxy
}
log
}
# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
# and its logs for more details (uc logs caddy).
`,
},
{
name: "no containers with x-caddy configs",
containers: []store.ContainerRecord{
newContainerRecordWithPorts(
"api",
"10.210.0.3",
[]string{"api.example.com:8080/http"},
"test-machine-id",
),
},
want: testCaddyfileHeader + `
# Sites generated from service ports.
http://api.example.com {
reverse_proxy 10.210.0.3:8080 {
import common_proxy
}
log
}
# NOTE: User-defined configs for services were skipped because Caddy is not running on this machine
# (not accessible via the shared admin socket /run/uncloud/caddy/admin.sock) or the latest
# generated config is invalid. Please check the service 'caddy' is running (uc inspect caddy)
# and its logs for more details (uc logs caddy).
`,
},
}
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Validator is not expected to be called in these tests.
generator := NewCaddyfileGenerator("test-machine-id", "test-machine", nil, nil)
config, err := generator.Generate(ctx, tt.containers, false)
require.NoError(t, err)
assert.Equal(t, tt.want, normaliseGeneratedTimestamp(config), "Generated Caddyfile doesn't match")
})
}
}
func newContainerRecordWithPorts(serviceName, ip string, ports []string, machineID string) store.ContainerRecord {
portsLabel := strings.Join(ports, ",")
return store.ContainerRecord{
@@ -1037,3 +1093,18 @@ func newContainerRecordWithPorts(serviceName, ip string, ports []string, machine
MachineID: machineID,
}
}
// caddyContainerFromTestContainers returns the local Caddy container from the provided test containers,
// or creates a default one if not found.
func caddyContainerFromTestContainers(records []store.ContainerRecord) api.ServiceContainer {
if caddyCtr := selectLocalCaddyContainer(records, "test-machine-id"); caddyCtr != nil {
return *caddyCtr
}
return newContainerRecordWithCaddyConfig(
"caddy",
"10.210.0.1",
"",
"test-machine-id",
time.Now(),
).Container
}
internal/machine/caddyconfig/client.go
+12 -8
View File
@@ -3,7 +3,6 @@ package caddyconfig
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
@@ -26,15 +25,15 @@ func NewCaddyAdminClient(socketPath string) *CaddyAdminClient {
client: &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", socketPath)
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", socketPath)
},
},
},
}
}
// IsAvailable checks if the local Caddy instance is listening on the admin socket.
// IsAvailable checks whether a Caddy process accepts connections at the admin socket.
func (c *CaddyAdminClient) IsAvailable() bool {
conn, err := net.DialTimeout("unix", c.socketPath, 1*time.Second)
// A stale socket file left over from a crashed Caddy container returns ECONNREFUSED so this is correctly handled
@@ -80,15 +79,16 @@ func (c *CaddyAdminClient) Adapt(ctx context.Context, caddyfile string) (string,
// If the response is a 400 Bad Request, try to parse the error message from it.
if resp.StatusCode == http.StatusBadRequest {
var apiError caddy.APIError
if err = json.Unmarshal(body, &apiError); err == nil {
return "", errors.New(apiError.Message)
if err = json.Unmarshal(body, &apiError); err == nil && apiError.Message != "" {
return "", &InvalidCaddyfileError{Message: apiError.Message}
}
return "", &InvalidCaddyfileError{Message: string(body)}
}
return "", errors.New(string(body))
return "", fmt.Errorf("adapt request failed: HTTP %d: %s", resp.StatusCode, string(body))
}
// Load loads a Caddyfile configuration into the Caddy instance running on the machine.
// Load loads a Caddyfile configuration into the Caddy instance.
// Due to a Caddy bug (https://github.com/caddyserver/caddy/issues/7246), we first adapt the Caddyfile to JSON
// and then load the JSON config to get proper error handling.
func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error {
@@ -96,7 +96,11 @@ func (c *CaddyAdminClient) Load(ctx context.Context, caddyfile string) error {
if err != nil {
return fmt.Errorf("adapt Caddyfile to JSON config: %w", err)
}
return c.LoadJSON(ctx, jsonConfig)
}
// LoadJSON loads a JSON configuration into the Caddy instance.
func (c *CaddyAdminClient) LoadJSON(ctx context.Context, jsonConfig string) error {
req, err := http.NewRequestWithContext(ctx, "POST", "http://localhost/load", strings.NewReader(jsonConfig))
if err != nil {
return fmt.Errorf("create load request: %w", err)
internal/machine/caddyconfig/client_test.go
+54
View File
@@ -0,0 +1,54 @@
package caddyconfig
import (
"context"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testAdminServer(t *testing.T, handler http.HandlerFunc) string {
t.Helper()
// macOS limits Unix socket paths to 104 bytes. t.TempDir can exceed that.
dir, err := os.MkdirTemp("/tmp", "uc-caddy-")
require.NoError(t, err)
t.Cleanup(func() {
_ = os.RemoveAll(dir)
})
socketPath := filepath.Join(dir, "admin.sock")
listener, err := net.Listen("unix", socketPath)
require.NoError(t, err)
server := httptest.NewUnstartedServer(handler)
_ = server.Listener.Close()
server.Listener = listener
server.Start()
t.Cleanup(server.Close)
return socketPath
}
func TestCaddyAdminClient_Validate(t *testing.T) {
var status atomic.Int64
status.Store(http.StatusBadRequest)
socket := testAdminServer(t, func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(int(status.Load()))
_, _ = w.Write([]byte(`{"message":"invalid config"}`))
})
client := NewCaddyAdminClient(socket)
err := client.Validate(context.Background(), "invalid")
var invalid *InvalidCaddyfileError
assert.ErrorAs(t, err, &invalid)
status.Store(http.StatusInternalServerError)
err = client.Validate(context.Background(), "invalid")
assert.Error(t, err)
assert.NotErrorAs(t, err, &invalid)
}
internal/machine/caddyconfig/controller.go
+197 -106
View File
@@ -2,7 +2,7 @@ package caddyconfig
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/netip"
@@ -10,6 +10,7 @@ import (
"path/filepath"
"slices"
"strings"
"time"
"github.com/psviderski/uncloud/internal/fs"
"github.com/psviderski/uncloud/internal/machine/store"
@@ -20,27 +21,40 @@ const (
CaddyServiceName = "caddy"
CaddyGroup = "uncloud"
VerifyPath = "/.uncloud-verify"
// periodicReconciliationInterval is the interval at which the controller reconciles the Caddyfile
// even if no container changes are observed.
periodicReconciliationInterval = 30 * time.Second
)
// Controller monitors container changes in the cluster store and generates a configuration file for Caddy reverse
// proxy. The generated configuration allows Caddy to route external traffic to service containers across the internal
// network.
// Controller keeps the local Caddy reverse proxy in sync with container state from the cluster store. It writes a
// Caddyfile that routes external traffic to healthy service containers across the internal network.
//
// Current Caddy deployments share one Caddyfile and one admin socket per machine. The controller must preserve a
// saved full Caddyfile when Caddy is unavailable because that file may be needed to restart or roll back a container.
// It writes a reduced bootstrap config only when no full file exists, including when an older offline file must be
// replaced. The shared layout cannot give overlapping Caddy revisions separate configurations.
type Controller struct {
machineID string
caddyfilePath string
service *Service
generator *CaddyfileGenerator
client *CaddyAdminClient
store *store.Store
log *slog.Logger
// lastFingerprint caches the fingerprint of the containers used to generate the latest successfully loaded
// Caddyfile. nil means it hasn't been loaded yet or the last load failed.
// Tests use their own group so they can exercise real file publication without the daemon's uncloud group.
fileGroup string
// lastFingerprint records the healthy application-container inputs used for the last full Caddyfile that was
// successfully loaded into Caddy and saved to disk. It excludes the Caddy container, whose identity, start time,
// and global config are tracked separately below.
lastFingerprint []containerFingerprint
// lastCaddyfile caches the last generated Caddyfile.
lastCaddyfile string
lastCaddyCtrID string
lastStartedAt string
lastGlobal string
lastSavedBody string
}
// containerFingerprint is the subset of container data that the Caddyfile generator depends on.
// Comparing fingerprints lets the controller skip no-op regenerations.
// containerFingerprint contains container identity and routing inputs that can change the generated Caddyfile.
// It excludes incidental Docker state so unrelated container updates do not cause a Caddy reload.
type containerFingerprint struct {
ID string
IP netip.Addr
@@ -56,7 +70,8 @@ func (f containerFingerprint) Equal(other containerFingerprint) bool {
f.CaddyConfig == other.CaddyConfig
}
func NewController(machineID, configDir, adminSock string, store *store.Store) (*Controller, error) {
func NewController(machineID string, service *Service, adminSock string, store *store.Store) (*Controller, error) {
configDir := service.configDir
if err := os.MkdirAll(configDir, 0o750); err != nil {
return nil, fmt.Errorf("create directory for Caddy configuration '%s': %w", configDir, err)
}
@@ -67,16 +82,21 @@ func NewController(machineID, configDir, adminSock string, store *store.Store) (
log := slog.With("component", "caddy-controller")
client := NewCaddyAdminClient(adminSock)
// generator is initialised by Run() once the machine name is resolved from the store.
// The generator is initialised by Run() after resolving the machine name from the store.
return &Controller{
machineID: machineID,
caddyfilePath: filepath.Join(configDir, "Caddyfile"),
service: service,
client: client,
store: store,
log: log,
fileGroup: CaddyGroup,
}, nil
}
// Run reconciles on container changes and every periodicReconciliationInterval. Failed work is retried every second,
// but repeated retry failures are logged only during the periodic check.
// The periodic check also covers missed events and outstanding failures after the daemon restarts.
func (c *Controller) Run(ctx context.Context) error {
// Default the machine name to the machine ID so the Caddyfile header still carries a stable identifier if
// the store lookup fails.
@@ -95,13 +115,34 @@ func (c *Controller) Run(ctx context.Context) error {
}
c.log.Info("Subscribed to container changes in the cluster to generate Caddy configuration.")
containers = filterHealthyContainers(containers)
c.generateAndLoadCaddyfile(ctx, containers)
var retryC <-chan time.Time
periodic := time.NewTicker(periodicReconciliationInterval)
defer periodic.Stop()
// TODO: left for backward compatibility, remove later.
if err = c.generateJSONConfig(containers); err != nil {
c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
handleResult := func(err error, logFailure bool) {
c.service.setReconciliationResult(err)
if err == nil {
retryC = nil
return
}
if logFailure {
c.log.Error("Failed to reconcile Caddy configuration, will retry.", "err", err)
}
retryC = time.After(time.Second)
}
reconcile := func(logFailure bool) {
ctrs, err := c.store.ListContainers(ctx, store.ListOptions{})
if err != nil {
err = fmt.Errorf("list containers: %w", err)
} else {
err = c.generateAndLoadCaddyfile(ctx, ctrs)
}
handleResult(err, logFailure)
}
// The subscription supplies an initial snapshot, so the first attempt need not wait for a change event.
handleResult(c.generateAndLoadCaddyfile(ctx, containers), true)
for {
select {
@@ -111,25 +152,18 @@ func (c *Controller) Run(ctx context.Context) error {
}
c.log.Debug("Cluster containers changed, regenerating Caddy configuration.")
containers, err = c.store.ListContainers(ctx, store.ListOptions{})
if err != nil {
c.log.Error("Failed to list containers.", "err", err)
continue
}
containers = filterHealthyContainers(containers)
c.generateAndLoadCaddyfile(ctx, containers)
// TODO: left for backward compatibility, remove later.
if err = c.generateJSONConfig(containers); err != nil {
c.log.Error("Failed to generate Caddy JSON configuration to disk.", "err", err)
}
reconcile(true)
case <-periodic.C:
reconcile(true)
case <-retryC:
reconcile(false)
case <-ctx.Done():
return nil
}
}
}
// filterHealthyContainers filters out unhealthy and hook containers.
// filterHealthyContainers filters out unhealthy, hook, and caddy containers.
// TODO: Filters out containers from this machine that are likely unavailable. The availability can be determined
// by the cluster membership state of the machine that the container is running on. Implement machine membership
// check using Corrossion Admin client.
@@ -139,6 +173,9 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain
if cr.Container.IsHook() {
continue
}
if cr.Container.ServiceName() == CaddyServiceName {
continue
}
if cr.Container.Healthy() {
healthy = append(healthy, cr)
}
@@ -146,65 +183,96 @@ func filterHealthyContainers(containers []store.ContainerRecord) []store.Contain
return healthy
}
// generateAndLoadCaddyfile regenerates the Caddyfile from the given containers and loads it into the local Caddy
// if available.
func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) {
// Check if Caddy is available before attempting to generate and load config.
caddyAvailable := c.client.IsAvailable()
// Skip regeneration when Caddy is available and the containers since the last successful load haven't changed.
// When Caddy is unavailable we still regenerate to keep the Caddyfile on disk updated.
fingerprint := fingerprintContainers(containers)
if caddyAvailable && slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) {
c.log.Debug("Caddy configuration is unchanged.", "path", c.caddyfilePath)
return
// generateAndLoadCaddyfile reconciles the shared Caddyfile for the selected local Caddy container. A full candidate
// reaches disk only after Caddy accepts it. Without an admin endpoint, the controller keeps a saved full file intact
// and writes a bootstrap config only when the file is absent or already a bootstrap.
func (c *Controller) generateAndLoadCaddyfile(ctx context.Context, containers []store.ContainerRecord) error {
caddyCtr := selectLocalCaddyContainer(containers, c.machineID)
if caddyCtr == nil {
// Caddy is not running locally which means there is no reliable source for the global config, skipping.
return nil
}
caddyfile, err := c.generator.Generate(ctx, containers, caddyAvailable)
saved, readErr := os.ReadFile(c.caddyfilePath)
if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
return fmt.Errorf("read saved Caddyfile: %w", readErr)
}
haveSaved := readErr == nil
healthyCtrs := filterHealthyContainers(containers)
fingerprint := fingerprintContainers(healthyCtrs)
if !caddyCtr.State.Running || !c.client.IsAvailable() {
if haveSaved && !isBootstrapCaddyfile(string(saved)) {
// Caddy may need this file to restart or roll back. A hosts-only replacement could drop
// storage or other global settings while the admin API is unavailable.
if caddyCtr.State.Running {
return fmt.Errorf("caddy admin socket unavailable, preserving saved Caddyfile")
}
return nil
}
bootstrap, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, true)
if err != nil {
c.log.Error("Failed to generate Caddyfile configuration.", "err", err)
return
return fmt.Errorf("generate Caddy bootstrap config: %w", err)
}
if err = c.writeCaddyfileIfChanged(bootstrap); err != nil {
return fmt.Errorf("save Caddy bootstrap config: %w", err)
}
if caddyCtr.State.Running {
return fmt.Errorf("caddy admin socket unavailable, bootstrap config saved")
}
return nil
}
if !caddyAvailable {
// Caddy is not running so the generated Caddyfile should not include user-defined configs thus must be valid.
// It's safe to write the config to disk so that when Caddy is deployed on this machine, it can pick it up.
if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
c.log.Error("Failed to write Caddyfile to disk.", "err", err)
return
}
c.log.Debug("Caddy is not running on this machine, skipping configuration load.", "path", c.caddyfilePath)
return
if haveSaved &&
caddyfileBody(string(saved)) == c.lastSavedBody &&
!isBootstrapCaddyfile(string(saved)) &&
c.lastCaddyCtrID == caddyCtr.ID &&
c.lastStartedAt == caddyCtr.State.StartedAt &&
c.lastGlobal == caddyCtr.ServiceSpec.CaddyConfig() &&
slices.EqualFunc(fingerprint, c.lastFingerprint, containerFingerprint.Equal) {
// No changes to the inputs that affect the generated Caddyfile, so no reload is needed.
return nil
}
// Caddy is available, try to load the config which may fail if the config is invalid. Generally, a config can
// pass the adaptation/validation step but still fail to load, for example, if it references resources that are
// not available.
caddyfile, err := c.generator.Generate(ctx, *caddyCtr, healthyCtrs, false)
if err != nil {
return fmt.Errorf("generate Caddyfile: %w", err)
}
// Try to load the config which may fail if the config is invalid. Generally, a config can pass
// the adaptation/validation step but still fail to load, for example, if it references resources
// that are not available.
if err = c.client.Load(ctx, caddyfile); err != nil {
c.log.Error("Failed to load new Caddy configuration into local Caddy instance.",
"err", err, "path", c.caddyfilePath)
// Mark the cache stale so the next container change retries the load even if the container set is unchanged.
c.lastFingerprint = nil
// Don't write invalid config to disk.
return
return fmt.Errorf("load Caddyfile: %w", err)
}
if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
return fmt.Errorf("save loaded Caddyfile: %w", err)
}
c.lastFingerprint = fingerprint
c.lastCaddyCtrID = caddyCtr.ID
c.lastStartedAt = caddyCtr.State.StartedAt
c.lastGlobal = caddyCtr.ServiceSpec.CaddyConfig()
c.lastSavedBody = caddyfileBody(caddyfile)
c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath)
// Config loaded successfully, now write it to disk.
if err = c.writeCaddyfileIfChanged(caddyfile); err != nil {
c.log.Error("Failed to write Caddyfile to disk after successful load.", "err", err)
// Config is already loaded in Caddy, so this is not critical. The next regeneration retries the disk write.
return
return nil
}
c.log.Info("New Caddy configuration loaded into local Caddy instance.", "path", c.caddyfilePath)
// isBootstrapCaddyfile distinguishes a reduced startup file from a full file that must survive Caddy outage.
// It also recognizes the older daemon's offline file. That file omitted every x-caddy block, including globals,
// so treating it as a protected full file would preserve the configuration-loss bug:
// https://github.com/psviderski/uncloud/issues/412
func isBootstrapCaddyfile(caddyfile string) bool {
return strings.Contains(caddyfile, bootstrapMarker) || strings.Contains(caddyfile, legacyBootstrapMarker)
}
// fingerprintContainers returns a fingerprint of containers that the Caddyfile generator depends on.
func fingerprintContainers(containers []store.ContainerRecord) []containerFingerprint {
fingerprints := make([]containerFingerprint, len(containers))
for i, cr := range containers {
// Ignore ports parsing error as not much we can do about it. The generator just logs them and continues.
// Ignore ports parsing error as not much we can do about it. The generator just logs them and skips
// the container.
ports, _ := cr.Container.ServicePorts()
fingerprints[i] = containerFingerprint{
ID: cr.Container.ID,
@@ -220,21 +288,71 @@ func fingerprintContainers(containers []store.ContainerRecord) []containerFinger
return fingerprints
}
// writeCaddyfileIfChanged writes the Caddyfile content to disk with proper permissions only if its body differs
// from the last successfully written content. The first line of the Caddyfile carries a generation timestamp that
// changes on every regeneration, so it's excluded from the comparison to avoid redundant writes.
// selectLocalCaddyContainer chooses which local Caddy container supplies global Caddy config. A running container
// takes precedence over a newer stopped replacement, even if the running container is unhealthy. If two revisions
// run at once, the shared admin socket does not identify its owner, so this choice remains best effort.
func selectLocalCaddyContainer(records []store.ContainerRecord, machineID string) *api.ServiceContainer {
var selected *api.ServiceContainer
for _, cr := range records {
ctr := cr.Container
if cr.MachineID != machineID || ctr.ServiceName() != CaddyServiceName || ctr.IsHook() {
continue
}
if selected == nil {
selected = &ctr
continue
}
if ctr.State.Running != selected.State.Running {
if ctr.State.Running {
selected = &ctr
}
continue
}
if ctr.CreatedTime().Compare(selected.CreatedTime()) > 0 {
selected = &ctr
}
}
return selected
}
// writeCaddyfileIfChanged atomically replaces the saved Caddyfile only if its body differs from the new content.
func (c *Controller) writeCaddyfileIfChanged(caddyfile string) error {
if caddyfileBody(caddyfile) == caddyfileBody(c.lastCaddyfile) {
saved, err := os.ReadFile(c.caddyfilePath)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("read Caddyfile '%s': %w", c.caddyfilePath, err)
}
if err == nil && caddyfileBody(caddyfile) == caddyfileBody(string(saved)) {
return nil
}
if err := os.WriteFile(c.caddyfilePath, []byte(caddyfile), 0o640); err != nil {
return fmt.Errorf("write Caddyfile to file '%s': %w", c.caddyfilePath, err)
dir := filepath.Dir(c.caddyfilePath)
tmp, err := os.CreateTemp(dir, ".Caddyfile-*")
if err != nil {
return fmt.Errorf("create temporary Caddyfile: %w", err)
}
if err := fs.Chown(c.caddyfilePath, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddyfile '%s': %w", c.caddyfilePath, err)
defer os.Remove(tmp.Name())
defer tmp.Close()
if err = tmp.Chmod(0o640); err != nil {
return fmt.Errorf("set temporary Caddyfile permissions: %w", err)
}
if err = fs.Chown(tmp.Name(), "", c.fileGroup); err != nil {
return fmt.Errorf("change owner of temporary Caddyfile: %w", err)
}
if _, err = tmp.WriteString(caddyfile); err != nil {
return fmt.Errorf("write temporary Caddyfile: %w", err)
}
if err = tmp.Sync(); err != nil {
return fmt.Errorf("sync temporary Caddyfile: %w", err)
}
if err = tmp.Close(); err != nil {
return fmt.Errorf("close temporary Caddyfile: %w", err)
}
if err = os.Rename(tmp.Name(), c.caddyfilePath); err != nil {
return fmt.Errorf("replace Caddyfile '%s': %w", c.caddyfilePath, err)
}
c.lastCaddyfile = caddyfile
return nil
}
@@ -247,30 +365,3 @@ func caddyfileBody(caddyfile string) string {
}
return caddyfile
}
func (c *Controller) generateJSONConfig(containers []store.ContainerRecord) error {
serviceContainers := make([]api.ServiceContainer, len(containers))
for i, cr := range containers {
serviceContainers[i] = cr.Container
}
config, err := GenerateJSONConfig(serviceContainers, c.machineID)
if err != nil {
return err
}
configBytes, err := json.MarshalIndent(config, "", " ")
if err != nil {
return fmt.Errorf("marshal Caddy configuration: %w", err)
}
configPath := filepath.Join(filepath.Dir(c.caddyfilePath), "caddy.json")
if err = os.WriteFile(configPath, configBytes, 0o640); err != nil {
return fmt.Errorf("write Caddy configuration to file '%s': %w", configPath, err)
}
if err = fs.Chown(configPath, "", CaddyGroup); err != nil {
return fmt.Errorf("change owner of Caddy configuration file '%s': %w", configPath, err)
}
return nil
}
internal/machine/caddyconfig/controller_test.go
+474
View File
@@ -1,14 +1,488 @@
package caddyconfig
import (
"context"
"io"
"log/slog"
"net/http"
"net/netip"
"os"
"os/user"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/docker/docker/api/types/container"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testController(t *testing.T, socketPath string) *Controller {
t.Helper()
group, err := user.LookupGroupId(strconv.Itoa(os.Getegid()))
require.NoError(t, err)
path := filepath.Join(t.TempDir(), "Caddyfile")
client := NewCaddyAdminClient(socketPath)
return &Controller{
machineID: "test-machine-id",
caddyfilePath: path,
generator: NewCaddyfileGenerator("test-machine-id", "test-machine", client, nil),
client: client,
log: slog.Default(),
fileGroup: group.Name,
}
}
type testCaddyAdmin struct {
mu sync.Mutex
adaptRequests []string
loadCount int
rejectAdaptContaining string
rejectLoad bool
}
func (a *testCaddyAdmin) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/adapt":
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
a.mu.Lock()
a.adaptRequests = append(a.adaptRequests, string(body))
reject := a.rejectAdaptContaining != "" && strings.Contains(string(body), a.rejectAdaptContaining)
a.mu.Unlock()
if reject {
http.Error(w, "invalid Caddyfile", http.StatusBadRequest)
return
}
_, _ = io.WriteString(w, `{"result":{}}`)
case "/load":
a.mu.Lock()
a.loadCount++
reject := a.rejectLoad
a.mu.Unlock()
if reject {
http.Error(w, "load rejected", http.StatusBadRequest)
}
default:
http.NotFound(w, r)
}
}
func (a *testCaddyAdmin) snapshot() ([]string, int) {
a.mu.Lock()
defer a.mu.Unlock()
return append([]string(nil), a.adaptRequests...), a.loadCount
}
func (a *testCaddyAdmin) setRejectLoad(reject bool) {
a.mu.Lock()
a.rejectLoad = reject
a.mu.Unlock()
}
func TestController_GenerateAndLoadCaddyfile(t *testing.T) {
t.Run("keeps the saved file when there is no local Caddy container", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
saved := "# previous full Caddyfile\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ remote }",
"other-machine", time.Now())
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{remote}))
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
})
t.Run("does not publish a bootstrap with an invalid global template", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{{upstreams",
"test-machine-id", time.Now())
require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(),
[]store.ContainerRecord{caddyCtr}), "render template")
_, err := os.Stat(controller.caddyfilePath)
assert.ErrorIs(t, err, os.ErrNotExist)
})
t.Run("bootstraps unhealthy Caddy with its globals and healthy routes", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"}
newerStopped := newContainerRecordWithCaddyConfig("caddy", "10.210.0.6", "{\n\tstorage replacement\n}",
"test-machine-id", time.Now().Add(time.Minute))
newerStopped.Container.State.Running = false
app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"},
"test-machine-id")
app.Container.ServiceSpec.Caddy = &api.CaddySpec{Config: "custom.example.com { respond app }"}
unhealthy := newContainerRecordWithPorts("unhealthy", "10.210.0.4",
[]string{"unhealthy.example.com:8080/http"}, "test-machine-id")
unhealthy.Container.State.Health = &container.Health{Status: "unhealthy"}
hook := newContainerRecordWithPorts("hook", "10.210.0.5",
[]string{"hook.example.com:8080/http"}, "test-machine-id")
hook.Container.Config.Labels[api.LabelHook] = "pre-deploy"
err := controller.generateAndLoadCaddyfile(context.Background(),
[]store.ContainerRecord{newerStopped, caddyCtr, app, unhealthy, hook})
require.ErrorContains(t, err, "admin socket unavailable")
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
assert.Contains(t, string(saved), "storage uncloud")
assert.NotContains(t, string(saved), "storage replacement")
assert.Contains(t, string(saved), "app.example.com")
assert.NotContains(t, string(saved), "custom.example.com")
assert.NotContains(t, string(saved), "unhealthy.example.com")
assert.NotContains(t, string(saved), "hook.example.com")
})
t.Run("replaces a legacy offline file with a bootstrap containing globals", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
legacy := "# generated by older daemon\nhttp:// { respond ok }\n" + legacyBootstrapMarker + "\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(legacy), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr})
require.ErrorContains(t, err, "admin socket unavailable")
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), "storage uncloud")
assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
assert.NotContains(t, string(saved), legacyBootstrapMarker)
})
t.Run("saves a bootstrap for a stopped Caddy container with no file", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
caddyCtr.Container.State.Running = false
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr}))
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), bootstrapMarker+caddyCtr.Container.ID)
assert.Contains(t, string(saved), "storage uncloud")
})
for _, tc := range []struct {
name string
running bool
}{
{name: "preserves a full file when the admin socket is unavailable", running: true},
{name: "preserves a stopped container's full file", running: false},
} {
t.Run(tc.name, func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage new\n}",
"test-machine-id", time.Now())
caddyCtr.Container.State.Running = tc.running
app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"new.example.com:8080/http"},
"test-machine-id")
err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app})
if tc.running {
require.ErrorContains(t, err, "admin socket unavailable")
} else {
require.NoError(t, err)
}
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
})
}
t.Run("loads a full file and reloads after Caddy restarts", func(t *testing.T) {
admin := &testCaddyAdmin{}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
caddyCtr.Container.State.Health = &container.Health{Status: "unhealthy"}
app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "app.example.com { respond app }",
"test-machine-id", time.Now())
records := []store.ContainerRecord{caddyCtr, app}
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), "storage uncloud")
assert.Contains(t, string(saved), "app.example.com")
assert.NotContains(t, string(saved), bootstrapMarker)
adapted, loads := admin.snapshot()
require.NotEmpty(t, adapted)
assert.Contains(t, adapted[0], "storage uncloud")
assert.Equal(t, 1, loads)
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
_, loads = admin.snapshot()
assert.Equal(t, 1, loads, "unchanged inputs should not reload Caddy")
records[0].Container.State.StartedAt = time.Now().UTC().Format(time.RFC3339Nano)
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
_, loads = admin.snapshot()
assert.Equal(t, 2, loads, "a restarted Caddy process needs the full configuration")
})
t.Run("reloads when an application route changes", func(t *testing.T) {
admin := &testCaddyAdmin{}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
app := newContainerRecordWithPorts("web", "10.210.0.3", []string{"app.example.com:8080/http"},
"test-machine-id")
records := []store.ContainerRecord{caddyCtr, app}
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
_, loads := admin.snapshot()
assert.Equal(t, 1, loads)
records[1] = newContainerRecordWithPorts("web", "10.210.0.4",
[]string{"app.example.com:8080/http"}, "test-machine-id")
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), "10.210.0.4:8080")
assert.NotContains(t, string(saved), "10.210.0.3:8080")
_, loads = admin.snapshot()
assert.Equal(t, 2, loads)
})
t.Run("rejects invalid globals without loading or replacing the saved file", func(t *testing.T) {
admin := &testCaddyAdmin{rejectAdaptContaining: "storage rejected"}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
saved := "{\n\tstorage uncloud\n}\n\nold.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage rejected\n}",
"test-machine-id", time.Now())
err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr})
require.ErrorContains(t, err, "validate user-defined global Caddy config")
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
_, loads := admin.snapshot()
assert.Zero(t, loads)
})
t.Run("skips invalid application custom config", func(t *testing.T) {
admin := &testCaddyAdmin{rejectAdaptContaining: "invalid.example.com"}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
invalid := newContainerRecordWithCaddyConfig("invalid", "10.210.0.3",
"invalid.example.com { respond bad }", "test-machine-id", time.Now())
valid := newContainerRecordWithCaddyConfig("valid", "10.210.0.4",
"valid.example.com { respond good }", "test-machine-id", time.Now())
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(),
[]store.ContainerRecord{caddyCtr, invalid, valid}))
saved, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(saved), "storage uncloud")
assert.Contains(t, string(saved), "valid.example.com")
assert.NotContains(t, string(saved), "invalid.example.com")
assert.Contains(t, string(saved), "Skipped invalid user-defined configs")
_, loads := admin.snapshot()
assert.Equal(t, 1, loads)
})
t.Run("keeps the saved file when application validation cannot complete", func(t *testing.T) {
var loads atomic.Int64
socket := testAdminServer(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/adapt":
http.Error(w, "adapter unavailable", http.StatusInternalServerError)
case "/load":
loads.Add(1)
default:
http.NotFound(w, r)
}
})
controller := testController(t, socket)
saved := "old.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "",
"test-machine-id", time.Now())
app := newContainerRecordWithCaddyConfig("web", "10.210.0.3", "new.example.com { respond new }",
"test-machine-id", time.Now())
err := controller.generateAndLoadCaddyfile(context.Background(), []store.ContainerRecord{caddyCtr, app})
require.ErrorContains(t, err, "validate Caddy config for service")
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
assert.Zero(t, loads.Load())
})
t.Run("retries a rejected load without changing inputs", func(t *testing.T) {
admin := &testCaddyAdmin{rejectLoad: true}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
records := []store.ContainerRecord{caddyCtr}
require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "load Caddyfile")
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
admin.setRejectLoad(false)
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
got, err = os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(got), "storage uncloud")
_, loads := admin.snapshot()
assert.Equal(t, 2, loads)
})
t.Run("retries saving after a successful load", func(t *testing.T) {
admin := &testCaddyAdmin{}
controller := testController(t, testAdminServer(t, admin.ServeHTTP))
group := controller.fileGroup
controller.fileGroup = "uncloud-test-group-that-does-not-exist"
saved := "{\n\tstorage old\n}\n\nold.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
records := []store.ContainerRecord{caddyCtr}
require.ErrorContains(t, controller.generateAndLoadCaddyfile(context.Background(), records), "save loaded Caddyfile")
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
controller.fileGroup = group
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
got, err = os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(got), "storage uncloud")
_, loads := admin.snapshot()
assert.Equal(t, 2, loads)
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
_, loads = admin.snapshot()
assert.Equal(t, 2, loads, "a loaded and saved revision should not reload")
})
t.Run("loads a saved bootstrap when the admin socket becomes available", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
caddyCtr := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{\n\tstorage uncloud\n}",
"test-machine-id", time.Now())
records := []store.ContainerRecord{caddyCtr}
require.Error(t, controller.generateAndLoadCaddyfile(context.Background(), records))
bootstrap, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Contains(t, string(bootstrap), bootstrapMarker)
admin := &testCaddyAdmin{}
socket := testAdminServer(t, admin.ServeHTTP)
controller.client = NewCaddyAdminClient(socket)
controller.generator = NewCaddyfileGenerator("test-machine-id", "test-machine", controller.client, nil)
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
full, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.NotContains(t, string(full), bootstrapMarker)
_, loads := admin.snapshot()
assert.Equal(t, 1, loads)
require.NoError(t, os.WriteFile(controller.caddyfilePath, bootstrap, 0o640))
require.NoError(t, controller.generateAndLoadCaddyfile(context.Background(), records))
_, loads = admin.snapshot()
assert.Equal(t, 2, loads, "a saved bootstrap is not an applied full configuration")
})
}
func TestController_WriteCaddyfileIfChanged(t *testing.T) {
t.Run("publishes a complete file with the configured group", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
caddyfile := "# Generated now\n{\n\tstorage uncloud\n}\n"
require.NoError(t, controller.writeCaddyfileIfChanged(caddyfile))
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, caddyfile, string(got))
info, err := os.Stat(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, os.FileMode(0o640), info.Mode().Perm())
})
t.Run("does not replace a file when only the timestamp changes", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
saved := "# Generated yesterday\nsite.example.com { respond ok }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
before, err := os.Stat(controller.caddyfilePath)
require.NoError(t, err)
require.NoError(t, controller.writeCaddyfileIfChanged(
"# Generated today\nsite.example.com { respond ok }\n"))
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
after, err := os.Stat(controller.caddyfilePath)
require.NoError(t, err)
assert.True(t, os.SameFile(before, after), "an unchanged body should not replace the file")
})
t.Run("preserves the previous file when publication fails", func(t *testing.T) {
controller := testController(t, filepath.Join(t.TempDir(), "missing.sock"))
controller.fileGroup = "uncloud-test-group-that-does-not-exist"
saved := "# previous\nsite.example.com { respond old }\n"
require.NoError(t, os.WriteFile(controller.caddyfilePath, []byte(saved), 0o640))
err := controller.writeCaddyfileIfChanged("# next\nsite.example.com { respond new }\n")
require.ErrorContains(t, err, "change owner of temporary Caddyfile")
got, err := os.ReadFile(controller.caddyfilePath)
require.NoError(t, err)
assert.Equal(t, saved, string(got))
tmpFiles, err := filepath.Glob(filepath.Join(filepath.Dir(controller.caddyfilePath), ".Caddyfile-*"))
require.NoError(t, err)
assert.Empty(t, tmpFiles)
})
}
func TestSelectLocalCaddyContainer(t *testing.T) {
t.Run("prefers a running unhealthy container over a newer stopped one", func(t *testing.T) {
older := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ older }",
"test-machine-id", time.Now().Add(-time.Hour))
older.Container.State.Health = &container.Health{Status: "unhealthy"}
newer := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ newer }",
"test-machine-id", time.Now())
newer.Container.State.Running = false
selected := selectLocalCaddyContainer([]store.ContainerRecord{newer, older}, "test-machine-id")
require.NotNil(t, selected)
assert.Equal(t, older.Container.ID, selected.ID)
})
t.Run("ignores remote and hook containers", func(t *testing.T) {
local := newContainerRecordWithCaddyConfig("caddy", "10.210.0.2", "{ local }",
"test-machine-id", time.Now().Add(-time.Hour))
remote := newContainerRecordWithCaddyConfig("caddy", "10.210.0.3", "{ remote }",
"other-machine", time.Now())
hook := newContainerRecordWithCaddyConfig("caddy", "10.210.0.4", "{ hook }",
"test-machine-id", time.Now())
hook.Container.Config.Labels[api.LabelHook] = "pre-deploy"
selected := selectLocalCaddyContainer([]store.ContainerRecord{remote, hook, local}, "test-machine-id")
require.NotNil(t, selected)
assert.Equal(t, local.Container.ID, selected.ID)
})
}
// TestContainerFingerprint_EqualCoversAllFields is a guard: when a field is added to containerFingerprint,
// Equal must also compare it. A mutation of any single field should flip equality to false. If this test fails
// after adding a field, update Equal to include it.
internal/machine/caddyconfig/jsonconfig.go
-141
View File
@@ -1,141 +0,0 @@
package caddyconfig
import (
"encoding/json"
"errors"
"fmt"
"maps"
"net/http"
"slices"
"strconv"
"time"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
"github.com/psviderski/uncloud/pkg/api"
)
func GenerateJSONConfig(containers []api.ServiceContainer, verifyResponse string) (*caddy.Config, error) {
httpHostUpstreams, httpsHostUpstreams := httpUpstreamsFromPorts(containers)
var warnings []caddyconfig.Warning
servers := make(map[string]*caddyhttp.Server)
servers["http"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPPort)},
// All http requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: append(
hostUpstreamsToRoutes(httpHostUpstreams, &warnings),
// Add a route to respond with a static verification response at the /.uncloud-verify path.
verificationRoute(verifyResponse, &warnings),
),
}
servers["https"] = &caddyhttp.Server{
Listen: []string{fmt.Sprintf(":%d", caddyhttp.DefaultHTTPSPort)},
// All https requests to this server are logged to the default logger.
Logs: &caddyhttp.ServerLogConfig{},
Routes: hostUpstreamsToRoutes(httpsHostUpstreams, &warnings),
}
httpApp := caddyhttp.App{
Servers: servers,
}
config := &caddy.Config{
AppsRaw: caddy.ModuleMap{
"http": caddyconfig.JSON(httpApp, &warnings),
},
}
var err error
if len(warnings) > 0 {
// warnings only contains errors from JSON marshaling, which are highly unlikely with correct code.
for _, w := range warnings {
err = errors.Join(err, errors.New(w.Message))
}
return nil, fmt.Errorf("marshal Caddy configuration: %w", err)
}
return config, nil
}
// hostUpstreamsToRoutes converts a map of hostnames to upstreams to a list of Caddy routes.
func hostUpstreamsToRoutes(hostUpstreams map[string][]string, warnings *[]caddyconfig.Warning) []caddyhttp.Route {
// Sort hostnames for deterministic output.
hostnames := slices.Collect(maps.Keys(hostUpstreams))
slices.Sort(hostnames)
routes := make([]caddyhttp.Route, 0, len(hostUpstreams))
for _, hostname := range hostnames {
upstreams := hostUpstreams[hostname]
upstreamPool := make([]*reverseproxy.Upstream, len(upstreams))
for i, upstream := range upstreams {
upstreamPool[i] = &reverseproxy.Upstream{
Dial: upstream,
}
}
handler := &reverseproxy.Handler{
HealthChecks: &reverseproxy.HealthChecks{
// Enable passive health checks to automatically detect unhealthy upstreams.
Passive: &reverseproxy.PassiveHealthChecks{
FailDuration: caddy.Duration(30 * time.Second),
},
},
LoadBalancing: &reverseproxy.LoadBalancing{
// Retry failed requests to skip over temporarily unavailable upstreams.
Retries: 3,
},
Upstreams: upstreamPool,
}
routes = append(routes, caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"host": caddyconfig.JSON(caddyhttp.MatchHost{hostname}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(handler, "handler", "reverse_proxy", warnings),
},
})
}
return routes
}
// verificationRoute returns a Caddy route that responds with the given static response at the /.uncloud-verify path.
func verificationRoute(response string, warnings *[]caddyconfig.Warning) caddyhttp.Route {
// Return the following route:
// {
// "match": [
// {
// "path": [
// "/.uncloud-verify"
// ]
// }
// ],
// "handle": [
// {
// "handler": "static_response",
// "body": "<response>",
// "status_code": 200
// }
// ]
// }
staticResponse := caddyhttp.StaticResponse{
StatusCode: caddyhttp.WeakString(strconv.Itoa(http.StatusOK)),
Body: response,
}
return caddyhttp.Route{
MatcherSetsRaw: caddyhttp.RawMatcherSets{
{
"path": caddyconfig.JSON(caddyhttp.MatchPath{VerifyPath}, warnings),
},
},
HandlersRaw: []json.RawMessage{
caddyconfig.JSONModuleObject(staticResponse, "handler", "static_response", warnings),
},
}
}
internal/machine/caddyconfig/jsonconfig_test.go
-378
View File
@@ -1,378 +0,0 @@
package caddyconfig
import (
"strings"
"testing"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/psviderski/uncloud/internal/machine/docker"
"github.com/psviderski/uncloud/pkg/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGenerateJSONConfig(t *testing.T) {
configWithoutServices := `{
"servers": {
"http": {
"listen": [":80"],
"routes": [{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}],
"logs": {}
},
"https": {
"listen": [":443"],
"logs": {}
}
}
}`
tests := []struct {
name string
containers []api.ServiceContainer
want string
wantErr bool
}{
{
name: "empty containers",
containers: []api.ServiceContainer{},
want: configWithoutServices,
wantErr: false,
},
{
name: "HTTP container",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8080"}]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
],
"logs": {}
},
"https": {
"listen": [":443"],
"logs": {}
}
}
}`,
wantErr: false,
},
{
name: "load balancing multiple containers",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "app.example.com:8080/http"),
newContainer("10.210.0.3", "app.example.com:8080/http"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"}
]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
],
"logs": {}
},
"https": {
"listen": [":443"],
"logs": {}
}
}
}`,
wantErr: false,
},
{
name: "HTTPS container",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "secure.example.com:8000/https"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
],
"logs": {}
},
"https": {
"listen": [":443"],
"routes": [
{
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [{"dial": "10.210.0.2:8000"}]
}]
}
],
"logs": {}
}
}
}`,
wantErr: false,
},
{
name: "mixed HTTP and HTTPS",
containers: []api.ServiceContainer{
newContainer("10.210.0.2",
"app.example.com:8080/http",
"web.example.com:8000/http"),
newContainer("10.210.0.3",
"app.example.com:8080/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.4",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
newContainer("10.210.0.5",
"app.example.com:8080/http",
"web.example.com:8000/http",
"secure.example.com:8888/https"),
},
want: `{
"servers": {
"http": {
"listen": [":80"],
"routes": [
{
"match": [{"host": ["app.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8080"},
{"dial": "10.210.0.3:8080"},
{"dial": "10.210.0.5:8080"}
]
}]
},
{
"match": [{"host": ["web.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.2:8000"},
{"dial": "10.210.0.4:8000"},
{"dial": "10.210.0.5:8000"}
]
}]
},
{
"match": [{"path": ["/.uncloud-verify"]}],
"handle": [{
"body": "verification-response-body",
"handler": "static_response",
"status_code": 200
}]
}
],
"logs": {}
},
"https": {
"listen": [":443"],
"routes": [
{
"match": [{"host": ["secure.example.com"]}],
"handle": [{
"handler": "reverse_proxy",
"health_checks": {
"passive": {
"fail_duration": 30000000000
}
},
"load_balancing": {
"retries": 3
},
"upstreams": [
{"dial": "10.210.0.3:8888"},
{"dial": "10.210.0.4:8888"},
{"dial": "10.210.0.5:8888"}
]
}]
}
],
"logs": {}
}
}
}`,
wantErr: false,
},
{
name: "container without uncloud network ignored",
containers: []api.ServiceContainer{
newContainerWithoutNetwork("ignored.example.com:8080/http"),
},
want: configWithoutServices,
wantErr: false,
},
{
name: "container with invalid port ignored",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "invalid-port"),
},
want: configWithoutServices,
wantErr: false,
},
{
name: "containers with unsupported protocols and host mode ignored",
containers: []api.ServiceContainer{
newContainer("10.210.0.2", "5000/tcp"),
newContainer("10.210.0.3", "5000/udp"),
newContainer("10.210.0.4", "80:8080/tcp@host"),
},
want: configWithoutServices,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := GenerateJSONConfig(tt.containers, "verification-response-body")
if tt.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
require.Len(t, config.AppsRaw, 1, "Expected one http app")
require.Contains(t, config.AppsRaw, "http", "Expected http app")
assert.JSONEq(t, tt.want, string(config.AppsRaw["http"]), "Generated Caddy app config doesn't match")
})
}
}
func newContainer(ip string, ports ...string) api.ServiceContainer {
portsLabel := strings.Join(ports, ",")
return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{
State: &container.State{
Running: true,
},
},
NetworkSettings: &container.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
docker.NetworkName: {
IPAddress: ip,
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
}}}
}
func newContainerWithoutNetwork(ports ...string) api.ServiceContainer {
portsLabel := strings.Join(ports, ",")
return api.ServiceContainer{Container: api.Container{InspectResponse: container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{
State: &container.State{
Running: true,
},
},
NetworkSettings: &container.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
"other-network": {
IPAddress: "172.17.0.2",
},
},
},
Config: &container.Config{
Labels: map[string]string{
api.LabelServicePorts: portsLabel,
},
},
}}}
}
internal/machine/caddyconfig/server.go
+11 -2
View File
@@ -2,6 +2,7 @@ package caddyconfig
import (
"context"
"errors"
"os"
"google.golang.org/grpc/codes"
@@ -22,18 +23,26 @@ func NewServer(service *Service) *Server {
return &Server{service: service}
}
// GetConfig retrieves the current Caddy configuration from the machine.
// GetConfig retrieves the saved Caddy configuration and the latest reconciliation error from the machine.
func (s *Server) GetConfig(ctx context.Context, _ *emptypb.Empty) (*pb.GetCaddyConfigResponse, error) {
caddyfile, modifiedAt, err := s.service.Caddyfile()
if err != nil {
if os.IsNotExist(err) {
if errors.Is(err, os.ErrNotExist) {
if lastErr := s.service.LastReconciliationError(); lastErr != nil {
return nil, status.Errorf(codes.NotFound, "%v; last Caddy config load failed: %v", err, lastErr)
}
return nil, status.Error(codes.NotFound, err.Error())
}
return nil, status.Error(codes.Internal, err.Error())
}
recErr := ""
if lastErr := s.service.LastReconciliationError(); lastErr != nil {
recErr = lastErr.Error()
}
return &pb.GetCaddyConfigResponse{
Caddyfile: caddyfile,
ModifiedAt: timestamppb.New(modifiedAt),
LastReconciliationError: recErr,
}, nil
}
internal/machine/caddyconfig/server_test.go
+62
View File
@@ -0,0 +1,62 @@
package caddyconfig
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
)
func TestServer_GetConfig(t *testing.T) {
t.Run("returns saved Caddyfile and reconciliation error", func(t *testing.T) {
dir := t.TempDir()
caddyfile := "example.com { respond ok }\n"
path := filepath.Join(dir, "Caddyfile")
require.NoError(t, os.WriteFile(path, []byte(caddyfile), 0o640))
info, err := os.Stat(path)
require.NoError(t, err)
service := NewService(dir)
service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud"))
config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
require.NoError(t, err)
assert.Equal(t, caddyfile, config.GetCaddyfile())
assert.True(t, info.ModTime().Equal(config.GetModifiedAt().AsTime()))
assert.Equal(t, "module not registered: caddy.storage.uncloud", config.GetLastReconciliationError())
service.setReconciliationResult(nil)
config, err = NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
require.NoError(t, err)
assert.Empty(t, config.GetLastReconciliationError())
})
t.Run("returns NotFound with reconciliation error when Caddyfile is absent", func(t *testing.T) {
service := NewService(t.TempDir())
service.setReconciliationResult(errors.New("module not registered: caddy.storage.uncloud"))
config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
assert.Nil(t, config)
assert.Equal(t, codes.NotFound, status.Code(err))
assert.ErrorContains(t, err, "Caddyfile")
assert.ErrorContains(t, err, "last Caddy config load failed: module not registered: caddy.storage.uncloud")
})
t.Run("returns NotFound without reconciliation error when Caddyfile is absent", func(t *testing.T) {
service := NewService(t.TempDir())
config, err := NewServer(service).GetConfig(context.Background(), &emptypb.Empty{})
assert.Nil(t, config)
assert.Equal(t, codes.NotFound, status.Code(err))
assert.NotContains(t, err.Error(), "last Caddy config load failed")
})
}
internal/machine/caddyconfig/service.go
+19 -1
View File
@@ -4,12 +4,15 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
)
// Service provides methods to interact with the Caddy configuration on the machine.
type Service struct {
configDir string
mu sync.RWMutex
lastReconciliationError error
}
// NewService creates a new Service instance with the specified Caddy configuration directory.
@@ -17,7 +20,8 @@ func NewService(configDir string) *Service {
return &Service{configDir: configDir}
}
// Caddyfile retrieves the current Caddy configuration (Caddyfile) from the machine's config directory.
// Caddyfile retrieves the saved Caddyfile from the machine's config directory. The saved file may differ from the
// running configuration if Caddy accepted a load but the subsequent write failed.
func (s *Service) Caddyfile() (string, time.Time, error) {
path := filepath.Join(s.configDir, "Caddyfile")
content, err := os.ReadFile(path)
@@ -33,3 +37,17 @@ func (s *Service) Caddyfile() (string, time.Time, error) {
return string(content), fileInfo.ModTime(), nil
}
// LastReconciliationError reports the most recent unsuccessful controller attempt, if any. A successful attempt
// clears it. An empty value does not prove that Caddy has loaded the saved Caddyfile.
func (s *Service) LastReconciliationError() error {
s.mu.RLock()
defer s.mu.RUnlock()
return s.lastReconciliationError
}
func (s *Service) setReconciliationResult(err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.lastReconciliationError = err
}
internal/machine/caddyconfig/service_test.go
+42
View File
@@ -0,0 +1,42 @@
package caddyconfig
import (
"errors"
"sync"
"testing"
"github.com/stretchr/testify/assert"
)
func TestService_LastReconciliationError(t *testing.T) {
service := NewService(t.TempDir())
assert.NoError(t, service.LastReconciliationError())
service.setReconciliationResult(errors.New("global Caddy config rejected"))
assert.EqualError(t, service.LastReconciliationError(), "global Caddy config rejected")
service.setReconciliationResult(nil)
assert.NoError(t, service.LastReconciliationError())
}
func TestService_LastReconciliationErrorConcurrent(t *testing.T) {
service := NewService(t.TempDir())
var wg sync.WaitGroup
for range 10 {
wg.Add(2)
go func() {
defer wg.Done()
for range 100 {
service.setReconciliationResult(errors.New("retry"))
service.setReconciliationResult(nil)
}
}()
go func() {
defer wg.Done()
for range 100 {
_ = service.LastReconciliationError()
}
}()
}
wg.Wait()
}
internal/machine/machine.go
+5 -2
View File
@@ -198,6 +198,8 @@ type Machine struct {
dockerServer *machinedocker.Server
// machineAPIServer handles API requests directly on this machine.
machineAPIServer *grpc.Server
// caddyService provides methods to interact with the Caddy configuration on the machine.
caddyService *caddyconfig.Service
// proxyDirector routes API requests to local or remote machines.
proxyDirector *apiproxy.Director
@@ -302,6 +304,7 @@ func NewMachine(config *Config) (*Machine, error) {
dockerService: dockerService,
clusterAPIServer: clusterAPIServer,
proxyDirector: proxyDirector,
caddyService: caddyconfig.NewService(config.CaddyConfigDir),
}
// Machine IP will only be available after the machine is initialised as a cluster member so wrap it in a function.
@@ -316,7 +319,7 @@ func NewMachine(config *Config) (*Machine, error) {
NetworkReady: m.IsNetworkReady,
WaitForNetworkReady: m.WaitForNetworkReady,
})
caddyServer := caddyconfig.NewServer(caddyconfig.NewService(config.CaddyConfigDir))
caddyServer := caddyconfig.NewServer(m.caddyService)
caddyStore, err := corroStore.Keyspace(caddystorage.Namespace)
if err != nil {
@@ -535,7 +538,7 @@ func (m *Machine) Run(ctx context.Context) error {
// It will also serve the current machine ID at /.uncloud-verify to verify Caddy reachability.
caddyconfigCtrl, err := caddyconfig.NewController(
m.state.ID,
m.config.CaddyConfigDir,
m.caddyService,
DefaultCaddyAdminSockPath,
m.store,
)
test/e2e/machine_test.go
+18
View File
@@ -10,6 +10,7 @@ import (
"github.com/psviderski/uncloud/api/pb"
"github.com/psviderski/uncloud/internal/ucind"
"github.com/psviderski/uncloud/pkg/api"
"github.com/psviderski/uncloud/pkg/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -456,6 +457,23 @@ func TestMachineOperations(t *testing.T) {
})
t.Run("remove machine clears container records from cluster store", func(t *testing.T) {
// The Caddy controller needs a local Caddy container to supply the global config before it can generate
// routes. Place it on the connected machine by ID because earlier tests rename that machine.
caddyDeployment, err := cli.NewCaddyDeployment("", "", api.Placement{
Machines: []string{c.Machines[0].ID},
})
require.NoError(t, err)
monitorPeriod := 5 * time.Second
caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &monitorPeriod
_, err = caddyDeployment.Run(ctx)
require.NoError(t, err)
t.Cleanup(func() {
err := cli.RemoveService(ctx, client.CaddyServiceName)
if err != nil && !errors.Is(err, api.ErrNotFound) {
assert.NoError(t, err)
}
})
// Deploy a global service with an HTTP ingress port so the auto-generated Caddyfile lists
// each container's IP as an upstream.
serviceName := "test-machine-rm-cleanup"
test/e2e/service_test.go
+23 -10
View File
@@ -27,6 +27,8 @@ import (
"github.com/psviderski/uncloud/pkg/client/deploy"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func newServiceID() string {
@@ -42,6 +44,9 @@ func TestDeployment(t *testing.T) {
clusterName := "ucind-test.deployment"
ctx := context.Background()
// Caddy may restart once while the controller writes its bootstrap file. Use the normal deployment monitor period
// instead of the zero-duration override used by most e2e tests.
caddyMonitorPeriod := 5 * time.Second
c, _ := createTestCluster(t, clusterName, ucind.CreateClusterOptions{Machines: 3}, true)
cli, cErr := c.Machines[0].Connect(ctx)
@@ -313,6 +318,7 @@ func TestDeployment(t *testing.T) {
deployment, err := cli.NewCaddyDeployment("", "", api.Placement{})
require.NoError(t, err)
deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
_, err = deployment.Run(ctx)
require.NoError(t, err)
@@ -345,6 +351,7 @@ func TestDeployment(t *testing.T) {
Machines: []string{c.Machines[0].Name},
})
require.NoError(t, err)
deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
image := deployment.Spec.Container.Image
_, err = deployment.Run(ctx)
@@ -361,6 +368,7 @@ func TestDeployment(t *testing.T) {
// Deploy to all machines without a placement constraint.
deployment, err = cli.NewCaddyDeployment(image, "", api.Placement{})
require.NoError(t, err)
deployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
_, err = deployment.Run(ctx)
require.NoError(t, err)
@@ -391,6 +399,13 @@ func TestDeployment(t *testing.T) {
}
})
// Without a Caddy container, the controller has no global config to pair with application configs. Keep any
// previously saved Caddyfile unchanged rather than publishing a new one without Caddy's global settings.
savedBefore, savedBeforeErr := cli.Caddy.GetConfig(ctx, nil)
if savedBeforeErr != nil {
require.Equal(t, codes.NotFound, status.Code(savedBeforeErr))
}
// First deploy a service with custom caddy config before caddy is deployed.
serviceCaddyfile := `test-custom-caddy-config.example.com {
reverse_proxy {{upstreams}} {
@@ -416,17 +431,13 @@ func TestDeployment(t *testing.T) {
require.NoError(t, err)
assertServiceMatchesSpec(t, svc, spec)
// Check that the generated Caddyfile contains a comment that user-define configs were skipped.
var config *pb.GetCaddyConfigResponse
require.Eventually(t, func() bool {
config, err = cli.Caddy.GetConfig(ctx, nil)
if err != nil {
return false
require.Never(t, func() bool {
current, currentErr := cli.Caddy.GetConfig(ctx, nil)
if savedBeforeErr != nil {
return currentErr == nil
}
return strings.Contains(config.Caddyfile, "# NOTE: User-defined configs for services were skipped")
}, 5*time.Second, 100*time.Millisecond)
assert.NotContains(t, config.Caddyfile, "test-custom-caddy-config.example.com {")
return currentErr == nil && current.Caddyfile != savedBefore.Caddyfile
}, 2*time.Second, 100*time.Millisecond)
// Now deploy caddy with custom config.
caddyCaddyfile := `{
@@ -438,6 +449,7 @@ myapp.example.com {
}`
caddyDeployment, err := cli.NewCaddyDeployment("", caddyCaddyfile, api.Placement{})
require.NoError(t, err)
caddyDeployment.Spec.UpdateConfig.MonitorPeriod = &caddyMonitorPeriod
_, err = caddyDeployment.Run(ctx)
require.NoError(t, err)
@@ -447,6 +459,7 @@ myapp.example.com {
assertServiceMatchesSpec(t, caddySvc, caddyDeployment.Spec)
// Wait for the Caddyfile to be regenerated with both custom configs.
var config *pb.GetCaddyConfigResponse
require.Eventually(t, func() bool {
config, err = cli.Caddy.GetConfig(ctx, nil)
if err != nil {