From f68c9859d922c53d0bc4442958c581eb0713e5f7 Mon Sep 17 00:00:00 2001 From: Pasha Sviderski Date: Thu, 17 Sep 2026 15:23:38 +1000 Subject: [PATCH] BREAKING CHANGE: move Uncloud API socket to /run/uncloud/api/uncloud.sock, activate it by systemd socket unit --- README.md | 10 +- caddystorage/module.go | 6 +- cmd/uc/machine/add.go | 2 +- cmd/uc/main.go | 5 +- cmd/uncloudd/dialstdio.go | 2 +- experiment/wg/connector.go | 2 +- internal/cli/cli.go | 2 +- internal/cli/config/connection_test.go | 4 +- internal/daemon/daemon.go | 64 ++++++- internal/daemon/daemon_test.go | 71 ++++++++ internal/machine/cluster.go | 2 +- internal/machine/constants/constants.go | 7 +- internal/machine/firewall/iptables_linux.go | 4 +- internal/machine/machine.go | 166 ++++++++++++------ pkg/client/connector/ssh.go | 8 +- pkg/client/connector/sshcli.go | 4 +- pkg/client/connector/tcp.go | 4 +- pkg/client/connector/unix.go | 4 +- scripts/docker/entrypoint.sh | 6 +- scripts/install.sh | 27 ++- scripts/uninstall.sh | 3 + .../2-getting-started/2-deploy-demo-app.md | 4 + .../3-concepts/1-clusters/1-connecting.md | 38 ++-- website/docs/7-cli-config-reference.md | 12 +- 24 files changed, 345 insertions(+), 112 deletions(-) create mode 100644 internal/daemon/daemon_test.go diff --git a/README.md b/README.md index fc5a618d..20e2277f 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,9 @@ $ uc machine init --name oracle-vm ubuntu@152.67.101.197 ✓ uncloudd binary installed: /usr/local/bin/uncloudd ⏳ Downloading uninstall script: https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/uninstall.sh ✓ uncloud-uninstall script installed: /usr/local/bin/uncloud-uninstall +✓ Systemd unit file created: /etc/systemd/system/uncloud.socket ✓ Systemd unit file created: /etc/systemd/system/uncloud.service +Created symlink /etc/systemd/system/sockets.target.wants/uncloud.socket → /etc/systemd/system/uncloud.socket. Created symlink /etc/systemd/system/multi-user.target.wants/uncloud.service → /etc/systemd/system/uncloud.service. ⏳ Starting Uncloud machine daemon (uncloud.service)... ✓ Uncloud machine daemon started. @@ -208,7 +210,9 @@ $ uc machine add --name hetzner-server root@5.223.45.199 ✓ uncloudd binary installed: /usr/local/bin/uncloudd ⏳ Downloading uninstall script: https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/uninstall.sh ✓ uncloud-uninstall script installed: /usr/local/bin/uncloud-uninstall +✓ Systemd unit file created: /etc/systemd/system/uncloud.socket ✓ Systemd unit file created: /etc/systemd/system/uncloud.service +Created symlink /etc/systemd/system/sockets.target.wants/uncloud.socket → /etc/systemd/system/uncloud.socket. Created symlink /etc/systemd/system/multi-user.target.wants/uncloud.service → /etc/systemd/system/uncloud.service. ⏳ Starting Uncloud machine daemon (uncloud.service)... ✓ Uncloud machine daemon started. @@ -319,9 +323,9 @@ I'm grateful to the following projects that inspired Uncloud's design and implem * [Tailscale](https://tailscale.com/) — for pioneering the vision of decentralised flat mesh networking with an amazing user experience that feels like magic. * [Talos Linux](https://github.com/siderolabs/talos) - and [KubeSpan](https://www.talos.dev/v1.10/talos-guides/network/kubespan/) — for the machine API design using - [grpc-proxy](https://github.com/siderolabs/grpc-proxy) and for its elegant approach to secure WireGuard-based overlay - networking with zero configuration. + and [KubeSpan](https://www.talos.dev/v1.10/talos-guides/network/kubespan/) — for inspiring Uncloud's API routing + design with [grpc-proxy](https://github.com/siderolabs/grpc-proxy) and for their elegant approach to secure + WireGuard-based overlay networking with zero configuration. * [Docker Swarm Classic](https://github.com/docker-archive/classicswarm) and [Rancher 1.x](http://rancher-com-website-main-elb-elb-1798790864.us-west-2.elb.amazonaws.com/docs/rancher/v1.6/en/) — for showing the power of simplicity and pragmatism in container orchestration and that not every problem needs the diff --git a/caddystorage/module.go b/caddystorage/module.go index d9fb128f..bca130e6 100644 --- a/caddystorage/module.go +++ b/caddystorage/module.go @@ -22,7 +22,7 @@ const ( // ModuleID is the Caddy module ID for Uncloud storage. ModuleID = "caddy.storage.uncloud" // DefaultSocketPath is the default path to the Uncloud API socket. - DefaultSocketPath = "/run/uncloud/uncloud.sock" + DefaultSocketPath = "/run/uncloud/api/uncloud.sock" // DefaultLockTTL is the default duration of a distributed lock lease. DefaultLockTTL = 20 * time.Second // lockCleanupTimeout bounds how long an unloaded module waits for active lock operations when cleaning up. @@ -37,7 +37,7 @@ func init() { // Storage implements a Caddy storage backend that uses an Uncloud cluster to store assets such as TLS certificates. type Storage struct { // Socket is the path to the Uncloud API socket. - // Defaults to /run/uncloud/uncloud.sock when not set. + // Defaults to /run/uncloud/api/uncloud.sock when not set. Socket string `json:"socket,omitempty"` // LockTTL is the duration of a distributed lock after which it expires if not renewed. Locks renew automatically // until unlocked. If an instance crashes or cannot renew, expiry allows another instance to acquire the stale lock. @@ -154,7 +154,7 @@ func (s *Storage) CertMagicStorage() (certmagic.Storage, error) { // // { // storage uncloud { -// socket /run/uncloud/uncloud.sock +// socket /run/uncloud/api/uncloud.sock // lock_ttl 20s // } // } diff --git a/cmd/uc/machine/add.go b/cmd/uc/machine/add.go index ff9332c2..792fcc4b 100644 --- a/cmd/uc/machine/add.go +++ b/cmd/uc/machine/add.go @@ -192,7 +192,7 @@ func add(ctx context.Context, uncli *cli.CLI, remoteMachine *cli.RemoteMachine, // Deploy a Caddy service container to the added machine. If caddy service is already deployed on other machines, // use the deployed image version. // NOTE: We use the cluster client to inspect and scale the Caddy service because the newly added machine may have - // issues accessing the Machine API of existing machines in the cluster. + // issues accessing the API of existing machines in the cluster. // See the issue for more details: https://github.com/psviderski/uncloud/issues/65. caddyImage := "" caddySvc, err := clusterClient.InspectService(ctx, client.CaddyServiceName) diff --git a/cmd/uc/main.go b/cmd/uc/main.go index e15f6d81..42a6e7fc 100644 --- a/cmd/uc/main.go +++ b/cmd/uc/main.go @@ -88,10 +88,11 @@ func main() { configPath := fs.ExpandHomeDir(opts.configPath) + // Make uc connect via the local Unix socket when running on a cluster machine. if opts.connect == "" { - if !fs.Exists(configPath) && fs.Exists(machine.DefaultUncloudSockPath) { + if !fs.Exists(configPath) && fs.Exists(machine.DefaultClusterAPISockPath) { conn = &config.MachineConnection{ - Unix: machine.DefaultUncloudSockPath, + Unix: machine.DefaultClusterAPISockPath, } } } diff --git a/cmd/uncloudd/dialstdio.go b/cmd/uncloudd/dialstdio.go index af13fbed..b1382a30 100644 --- a/cmd/uncloudd/dialstdio.go +++ b/cmd/uncloudd/dialstdio.go @@ -23,7 +23,7 @@ func newDialStdioCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&socketPath, "socket", machine.DefaultUncloudSockPath, + cmd.Flags().StringVar(&socketPath, "socket", machine.DefaultClusterAPISockPath, "Path to the Uncloud API socket") return cmd diff --git a/experiment/wg/connector.go b/experiment/wg/connector.go index 3fe0f418..3f89cf68 100644 --- a/experiment/wg/connector.go +++ b/experiment/wg/connector.go @@ -60,7 +60,7 @@ func (c *WireGuardConnector) Connect(ctx context.Context) (*grpc.ClientConn, err } endpoint := netip.AddrPortFrom(endpointAddr, DefaultEndpointPort) machineManagementIP := network.ManagementIP(machine.PublicKey) - machineAPIAddr := net.JoinHostPort(machineManagementIP.String(), strconv.Itoa(constants.MachineAPIPort)) + machineAPIAddr := net.JoinHostPort(machineManagementIP.String(), strconv.Itoa(constants.UncloudAPIPort)) tunCfg := &Config{ LocalAddress: c.user.ManagementIP(), diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 116faa74..e34720c2 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -603,7 +603,7 @@ func provisionOrConnectRemoteMachine( if remoteMachine.User != rootUser { // provisionMachine has just added the user to the uncloud group. Any SSH ControlMaster left over from // a previous uc invocation (e.g. a failed uc command against the uninitialised machine) still holds - // the old user groups and would deny access to /run/uncloud/uncloud.sock. Close the current session + // the old user groups and would deny access to /run/uncloud/api/uncloud.sock. Close the current session // if it exists so the next session picks up the new groups. conn.CloseControlMaster(ctx) } diff --git a/internal/cli/config/connection_test.go b/internal/cli/config/connection_test.go index b0f7d933..dddf1093 100644 --- a/internal/cli/config/connection_test.go +++ b/internal/cli/config/connection_test.go @@ -70,9 +70,9 @@ func TestMachineConnection_String(t *testing.T) { { name: "unix connection", conn: MachineConnection{ - Unix: "/run/uncloud/uncloud.sock", + Unix: "/run/uncloud/api/uncloud.sock", }, - want: "unix:///run/uncloud/uncloud.sock", + want: "unix:///run/uncloud/api/uncloud.sock", }, { name: "no connection", diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9f39cdf9..081b9685 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -2,24 +2,50 @@ package daemon import ( "context" + "errors" "fmt" "log/slog" + "net" + "github.com/coreos/go-systemd/activation" systemd "github.com/coreos/go-systemd/daemon" "github.com/psviderski/uncloud/internal/machine" ) +const systemdSocketUnit = "uncloud.socket" + type Daemon struct { machine *machine.Machine } func New(dataDir string) (*Daemon, error) { + listeners, err := activation.ListenersWithNames() + if err != nil { + return nil, fmt.Errorf("get systemd-activated sockets: %w", err) + } + + listener, err := selectActivatedListener(listeners) + if err != nil { + return nil, err + } + if listener != nil { + slog.Info("Using systemd-activated API socket.", "addr", listener.Addr().String()) + } + config := &machine.Config{ - DataDir: dataDir, + DataDir: dataDir, + ClusterAPISockPath: machine.DefaultClusterAPISockPath, + ClusterAPIListener: listener, } mach, err := machine.NewMachine(config) if err != nil { - return nil, fmt.Errorf("init machine: %w", err) + initErr := fmt.Errorf("init machine: %w", err) + if listener != nil { + if closeErr := listener.Close(); closeErr != nil { + initErr = errors.Join(initErr, fmt.Errorf("close systemd-activated socket: %w", closeErr)) + } + } + return nil, initErr } return &Daemon{ @@ -27,6 +53,40 @@ func New(dataDir string) (*Daemon, error) { }, nil } +// selectActivatedListener returns the systemd activated listener. If systemd supplied multiple listeners, +// it returns the one associated with systemdSocketUnit. +func selectActivatedListener(listeners map[string][]net.Listener) (net.Listener, error) { + var activated []net.Listener + for _, named := range listeners { + activated = append(activated, named...) + } + + if len(activated) == 0 { + return nil, nil + } + if len(activated) == 1 { + return activated[0], nil + } + + named, ok := listeners[systemdSocketUnit] + if !ok || len(named) != 1 { + return nil, fmt.Errorf("expected one systemd-activated socket named '%s', received %d", + systemdSocketUnit, len(named)) + } + + // Close all other unused listeners. + for name, unused := range listeners { + if name == systemdSocketUnit { + continue + } + for _, l := range unused { + _ = l.Close() + } + } + + return named[0], nil +} + func (d *Daemon) Run(ctx context.Context) error { slog.Info("Starting machine.") diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go new file mode 100644 index 00000000..c99f047d --- /dev/null +++ b/internal/daemon/daemon_test.go @@ -0,0 +1,71 @@ +package daemon + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSelectActivatedListener(t *testing.T) { + t.Run("no activated listeners", func(t *testing.T) { + listener, err := selectActivatedListener(nil) + require.NoError(t, err) + assert.Nil(t, listener) + }) + + t.Run("only listener regardless of name", func(t *testing.T) { + activated := testListener(t) + + listener, err := selectActivatedListener(map[string][]net.Listener{"another.socket": {activated}}) + require.NoError(t, err) + assert.Same(t, activated, listener) + }) + + t.Run("named listener when multiple", func(t *testing.T) { + activated := testListener(t) + extra := testListener(t) + + listener, err := selectActivatedListener(map[string][]net.Listener{ + systemdSocketUnit: {activated}, + "another.socket": {extra}, + }) + require.NoError(t, err) + assert.Same(t, activated, listener) + assert.Error(t, extra.Close()) + }) + + t.Run("multiple listeners without expected name", func(t *testing.T) { + first := testListener(t) + second := testListener(t) + + listener, err := selectActivatedListener(map[string][]net.Listener{ + "first.socket": {first}, + "second.socket": {second}, + }) + require.ErrorContains(t, err, "expected one systemd-activated socket") + assert.Nil(t, listener) + }) + + t.Run("multiple listeners under expected name", func(t *testing.T) { + first := testListener(t) + second := testListener(t) + + listener, err := selectActivatedListener(map[string][]net.Listener{ + systemdSocketUnit: {first, second}, + }) + require.ErrorContains(t, err, "expected one systemd-activated socket") + assert.Nil(t, listener) + }) +} + +func testListener(t *testing.T) net.Listener { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + return listener +} diff --git a/internal/machine/cluster.go b/internal/machine/cluster.go index c4368d65..da8228cb 100644 --- a/internal/machine/cluster.go +++ b/internal/machine/cluster.go @@ -176,7 +176,7 @@ func (cc *clusterController) Run(ctx context.Context) error { // Start the network API server before waiting for the store sync so the machine is reachable on the mesh // during the sync and can serve requests that don't depend on the store. // Assume the management IP can't be changed when the network is running. - apiAddr := net.JoinHostPort(cc.state.Network.ManagementIP.String(), strconv.Itoa(constants.MachineAPIPort)) + apiAddr := net.JoinHostPort(cc.state.Network.ManagementIP.String(), strconv.Itoa(constants.UncloudAPIPort)) listener, err := net.Listen("tcp", apiAddr) if err != nil { return fmt.Errorf("listen API port: %w", err) diff --git a/internal/machine/constants/constants.go b/internal/machine/constants/constants.go index 78b456bc..90af7491 100644 --- a/internal/machine/constants/constants.go +++ b/internal/machine/constants/constants.go @@ -1,8 +1,9 @@ package constants const ( - // MachineAPIPort is the port for the Machine API service on the management WireGuard network. - MachineAPIPort = 51000 - // UnregistryPort is the port for the embedded container registry listening on the machine IP. + // UncloudAPIPort is the TCP port on which each machine serves the Uncloud API over the management WireGuard + // network. + UncloudAPIPort = 51000 + // UnregistryPort is the TCP port for the embedded container registry listening on the machine IP. UnregistryPort = 51500 ) diff --git a/internal/machine/firewall/iptables_linux.go b/internal/machine/firewall/iptables_linux.go index 6a7e607d..e4dde066 100644 --- a/internal/machine/firewall/iptables_linux.go +++ b/internal/machine/firewall/iptables_linux.go @@ -48,12 +48,12 @@ func ConfigureIptablesChains(machineIP netip.Addr, wgPort int) error { } } - // Allow cluster machines to access Machine API via the management IPv6 WireGuard network. + // Allow cluster machines to access the Uncloud API via the management IPv6 WireGuard network. acceptMachineAPIRule := []string{ "-i", network.WireGuardInterfaceName, "-s", "fdcc::/16", "-p", "tcp", - "--dport", strconv.Itoa(constants.MachineAPIPort), + "--dport", strconv.Itoa(constants.UncloudAPIPort), "-j", "ACCEPT", } // Allow Corrosion gossip traffic from cluster machines via the management IPv6 WireGuard network. diff --git a/internal/machine/machine.go b/internal/machine/machine.go index cb446d4a..c678cf61 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -58,9 +58,13 @@ import ( ) const ( - DefaultMachineSockPath = "/run/uncloud/machine.sock" - DefaultUncloudSockPath = "/run/uncloud/uncloud.sock" - DefaultSockGroup = "uncloud" + // DefaultMachineAPISockPath is the default path to the Unix socket for API requests handled directly by this + // machine. + DefaultMachineAPISockPath = "/run/uncloud/machine.sock" + // DefaultClusterAPISockPath is the default path to the Unix socket for the client-facing API that routes requests + // across the cluster. + DefaultClusterAPISockPath = "/run/uncloud/api/uncloud.sock" + DefaultSockGroup = "uncloud" // DefaultCaddyAdminSockPath is the default path to the Caddy admin socket for validating the generated Caddy // reverse proxy configuration. DefaultCaddyAdminSockPath = "/run/uncloud/caddy/admin.sock" @@ -70,9 +74,15 @@ const ( type Config struct { // DataDir is the directory where the machine stores its persistent state. Default is /var/lib/uncloud. - DataDir string - MachineSockPath string - UncloudSockPath string + DataDir string + // MachineAPISockPath is the path to the Unix socket for API requests handled directly by this machine. + MachineAPISockPath string + // ClusterAPISockPath is the path to the Unix socket for the client-facing API that routes requests across the + // cluster. + ClusterAPISockPath string + // ClusterAPIListener is an optional pre-bound listener for the client-facing API that routes requests across the + // cluster. If set, Run takes ownership of it and closes it on shutdown or startup failure. + ClusterAPIListener net.Listener CorrosionDataDir string // CorrosionRunDir is the runtime directory for the corrosion service. @@ -104,11 +114,11 @@ func (c *Config) SetDefaults() (*Config, error) { if cfg.DataDir == "" { cfg.DataDir = "/var/lib/uncloud" } - if cfg.MachineSockPath == "" { - cfg.MachineSockPath = DefaultMachineSockPath + if cfg.MachineAPISockPath == "" { + cfg.MachineAPISockPath = DefaultMachineAPISockPath } - if cfg.UncloudSockPath == "" { - cfg.UncloudSockPath = DefaultUncloudSockPath + if cfg.ClusterAPISockPath == "" && cfg.ClusterAPIListener == nil { + cfg.ClusterAPISockPath = DefaultClusterAPISockPath } if cfg.DockerClient == nil { @@ -165,7 +175,7 @@ type Machine struct { config Config state *State - // started is closed when the machine is ready to serve requests on the local API server. + // started is closed when the machine and cluster API servers are ready to serve requests. started chan struct{} // initialised is closed when the machine is configured as a member of a cluster. initialised chan struct{} @@ -186,15 +196,14 @@ type Machine struct { // dockerService provides high-level operations for managing Docker containers. dockerService *machinedocker.Service dockerServer *machinedocker.Server - // localMachineServer is the gRPC server for the machine API listening on the local Unix socket. - localMachineServer *grpc.Server + // machineAPIServer handles API requests directly on this machine. + machineAPIServer *grpc.Server - // proxyDirector manages routing of gRPC requests between local and remote machine API servers. + // proxyDirector routes API requests to local or remote machines. proxyDirector *apiproxy.Director - // localProxyServer is the gRPC proxy server for the machine API listening on the local Unix socket. - // It proxies requests to the local or remote machine API servers depending on the request targets - // and aggregates responses. - localProxyServer *grpc.Server + // clusterAPIServer is the client-facing gRPC server that routes API requests across the cluster. It sends requests + // to one or more local or remote machines and aggregates their responses. + clusterAPIServer *grpc.Server // mu protects the Machine from concurrent reads and writes. mu sync.RWMutex @@ -269,10 +278,10 @@ func NewMachine(config *Config) (*Machine, error) { } dockerService := machinedocker.NewService(config.DockerClient, db) - // Init a local gRPC proxy server that proxies requests to the local or remote machine API servers. + // Init the client-facing API server that routes requests to local or remote machines. mapper := apiproxy.NewCorrosionMapper(corroStore) - proxyDirector := apiproxy.NewDirector(config.MachineSockPath, constants.MachineAPIPort, mapper) - localProxyServer := grpc.NewServer( + proxyDirector := apiproxy.NewDirector(config.MachineAPISockPath, constants.UncloudAPIPort, mapper) + clusterAPIServer := grpc.NewServer( grpc.ForceServerCodecV2(proxy.Codec()), grpc.UnaryInterceptor(grpcversion.ServerUnaryInterceptor), grpc.StreamInterceptor(grpcversion.ServerStreamInterceptor), @@ -291,7 +300,7 @@ func NewMachine(config *Config) (*Machine, error) { store: corroStore, cluster: c, dockerService: dockerService, - localProxyServer: localProxyServer, + clusterAPIServer: clusterAPIServer, proxyDirector: proxyDirector, } @@ -316,7 +325,7 @@ func NewMachine(config *Config) (*Machine, error) { caddyStorageServer := caddystorage.NewServer(caddyStore) leaseServer := distlockgrpc.NewServer(distlock.NewMemoryStore()) - m.localMachineServer = newGRPCServer(m, c, m.dockerServer, caddyServer, caddyStorageServer, leaseServer) + m.machineAPIServer = newGRPCServer(m, c, m.dockerServer, caddyServer, caddyStorageServer, leaseServer) if m.Initialised() { close(m.initialised) @@ -343,7 +352,7 @@ func newGRPCServer( return s } -// Started returns a channel that is closed when the machine is ready to serve requests on the local API server. +// Started returns a channel that is closed when the machine and cluster API servers are ready to serve requests. func (m *Machine) Started() <-chan struct{} { return m.started } @@ -395,21 +404,54 @@ func (m *Machine) Run(ctx context.Context) error { // Create a cancellable context for the Run method to allow stopping the machine gracefully. ctx, m.stop = context.WithCancel(ctx) + // Take over the systemd-activated socket for the cluster API if provided, otherwise bind to the configured + // Unix socket path. + clusterAPIListener := m.config.ClusterAPIListener + clusterAPISockPath := m.config.ClusterAPISockPath + if clusterAPIListener != nil { + if clusterAPIListener.Addr().Network() == "unix" { + clusterAPISockPath = clusterAPIListener.Addr().String() + } else { + clusterAPISockPath = "" + } + defer clusterAPIListener.Close() + } + + sockGID, err := socketGID() + if err != nil { + return err + } + if err = prepareUnixSocketDirectory(m.config.MachineAPISockPath, sockGID); err != nil { + return fmt.Errorf("prepare machine API Unix socket directory: %w", err) + } + // The cluster API socket activated by systemd may have been created with root group ownership. + // Apply the intended access permissions. + if clusterAPISockPath != "" { + if err = prepareUnixSocketDirectory(clusterAPISockPath, sockGID); err != nil { + return fmt.Errorf("prepare cluster API Unix socket directory: %w", err) + } + } + // Docker dependency is essential for the machine to function. Block until it's ready. if err := docker.WaitDaemonReady(ctx, m.config.DockerClient); err != nil { return fmt.Errorf("wait for Docker daemon: %w", err) } defer m.config.DockerClient.Close() - // Bind the local API listeners before starting the dependencies (e.g. corrosion) to not deal with the teardown + // Bind the API listeners before starting the dependencies (e.g. corrosion) to not deal with the teardown // on failure. - machineListener, err := listenUnixSocket(m.config.MachineSockPath) + machineAPIListener, err := sockets.NewUnixSocket(m.config.MachineAPISockPath, sockGID) if err != nil { - return fmt.Errorf("listen machine API unix socket %q: %w", m.config.MachineSockPath, err) + return fmt.Errorf("listen machine API Unix socket %q: %w", m.config.MachineAPISockPath, err) } - proxyListener, err := listenUnixSocket(m.config.UncloudSockPath) - if err != nil { - return fmt.Errorf("listen API proxy unix socket %q: %w", m.config.UncloudSockPath, err) + defer machineAPIListener.Close() + + if clusterAPIListener == nil { + clusterAPIListener, err = sockets.NewUnixSocket(clusterAPISockPath, sockGID) + if err != nil { + return fmt.Errorf("listen cluster API Unix socket %q: %w", clusterAPISockPath, err) + } + defer clusterAPIListener.Close() } // Configure and start the corrosion service on the loopback if the machine is not initialised as a cluster @@ -437,20 +479,20 @@ func (m *Machine) Run(ctx context.Context) error { // Use an errgroup to coordinate error handling and graceful shutdown of multiple machine components. errGroup, ctx := errgroup.WithContext(ctx) - // Start the local machine API server. + // Start the API server that handles requests directly on this machine. errGroup.Go(func() error { - slog.Info("Starting local machine API server.", "path", m.config.MachineSockPath) - if err := m.localMachineServer.Serve(machineListener); err != nil { - return fmt.Errorf("local machine API server failed: %w", err) + slog.Info("Starting machine API server.", "path", m.config.MachineAPISockPath) + if err := m.machineAPIServer.Serve(machineAPIListener); err != nil { + return fmt.Errorf("machine API server failed: %w", err) } return nil }) - // Start the local API proxy server. + // Start the client-facing API server that routes requests across the cluster. errGroup.Go(func() error { - slog.Info("Starting local API proxy server.", "path", m.config.UncloudSockPath) - if err := m.localProxyServer.Serve(proxyListener); err != nil { - return fmt.Errorf("local API proxy server failed: %w", err) + slog.Info("Starting cluster API server.", "path", clusterAPIListener.Addr().String()) + if err := m.clusterAPIServer.Serve(clusterAPIListener); err != nil { + return fmt.Errorf("cluster API server failed: %w", err) } return nil }) @@ -478,7 +520,7 @@ func (m *Machine) Run(ctx context.Context) error { slog.Info("Starting cluster controller.") // Update the proxy director's local address to the machine's management IP address, allowing - // the proxy to identify which requests should be proxied to the local machine API server. + // the proxy to identify which requests should be handled by the local API server. m.proxyDirector.UpdateLocalAddress(m.state.Network.ManagementIP.String()) proxyServer := grpc.NewServer( grpc.ForceServerCodecV2(proxy.Codec()), @@ -577,17 +619,17 @@ func (m *Machine) Run(ctx context.Context) error { // Shutdown goroutine. errGroup.Go(func() error { <-ctx.Done() - slog.Info("Stopping local machine API server.") + slog.Info("Stopping machine API server.") // TODO: implement timeout for graceful shutdown. - m.localMachineServer.GracefulStop() - slog.Info("Local machine API server stopped.") + m.machineAPIServer.GracefulStop() + slog.Info("Machine API server stopped.") - slog.Info("Stopping local API proxy server.") + slog.Info("Stopping cluster API server.") // TODO: implement timeout for graceful shutdown. - m.localProxyServer.GracefulStop() + m.clusterAPIServer.GracefulStop() // Close the proxy director to close all backend connections. m.proxyDirector.Close() - slog.Info("Local API proxy server stopped.") + slog.Info("Cluster API server stopped.") return nil }) @@ -616,38 +658,46 @@ func (m *Machine) Run(ctx context.Context) error { return err } -// listenUnixSocket creates a new Unix socket listener with the specified path. The socket file is created with 0660 -// access mode and uncloud group if the group is found, otherwise it falls back to the root group. -func listenUnixSocket(path string) (net.Listener, error) { +// socketGID returns the uncloud group ID, or the root group ID if the uncloud group does not exist. +func socketGID() (int, error) { gid := 0 // Fall back to the root group if the uncloud group is not found. group, err := user.LookupGroup(DefaultSockGroup) if err != nil { //goland:noinspection GoTypeAssertionOnErrors if _, ok := err.(user.UnknownGroupError); ok { slog.Info( - "Specified group not found, using root group for the API socket.", - "group", DefaultSockGroup, "path", path, + "Specified group not found, using root group for API sockets.", + "group", DefaultSockGroup, ) } else { - return nil, fmt.Errorf("lookup %q group ID (GID): %w", DefaultSockGroup, err) + return 0, fmt.Errorf("lookup %q group ID (GID): %w", DefaultSockGroup, err) } } else { gid, err = strconv.Atoi(group.Gid) if err != nil { - return nil, fmt.Errorf("parse %q group ID (GID) %q: %w", DefaultSockGroup, group.Gid, err) + return 0, fmt.Errorf("parse %q group ID (GID) %q: %w", DefaultSockGroup, group.Gid, err) } } - // Ensure the parent directory exists and has the correct group permissions. + return gid, nil +} + +// prepareUnixSocketDirectory ensures the socket's parent directory has the intended group and permissions. +func prepareUnixSocketDirectory(path string, gid int) error { parent, _ := filepath.Split(path) - if err = os.MkdirAll(parent, 0o750); err != nil { - return nil, fmt.Errorf("create directory %q: %w", parent, err) + if err := os.MkdirAll(parent, 0o750); err != nil { + return fmt.Errorf("create directory %q: %w", parent, err) } - if err = os.Chown(parent, -1, gid); err != nil { - return nil, fmt.Errorf("chown directory %q: %w", parent, err) + if err := os.Chown(parent, -1, gid); err != nil { + return fmt.Errorf("chown directory %q: %w", parent, err) + } + // MkdirAll preserves the mode of an existing directory. A Docker container may have created the bind-mount source + // first, so apply the intended mode explicitly. + if err := os.Chmod(parent, 0o750); err != nil { + return fmt.Errorf("chmod directory %q: %w", parent, err) } - return sockets.NewUnixSocket(path, gid) + return nil } func (m *Machine) configureCorrosion() error { diff --git a/pkg/client/connector/ssh.go b/pkg/client/connector/ssh.go index 3b19aa1c..4a3e51e7 100644 --- a/pkg/client/connector/ssh.go +++ b/pkg/client/connector/ssh.go @@ -35,7 +35,7 @@ func (cfg *SSHConnectorConfig) Destination() string { return dst } -// SSHConnector establishes a connection to the machine API through an SSH tunnel to the machine. +// SSHConnector establishes a connection to the Uncloud API through an SSH tunnel to the machine. type SSHConnector struct { config SSHConnectorConfig client *ssh.Client @@ -69,7 +69,7 @@ func (c *SSHConnector) Connect(ctx context.Context) (*grpc.ClientConn, error) { sockPath := c.config.SockPath if sockPath == "" { - sockPath = machine.DefaultUncloudSockPath + sockPath = machine.DefaultClusterAPISockPath } conn, err := grpc.NewClient( "unix://"+sockPath, @@ -83,7 +83,7 @@ func (c *SSHConnector) Connect(ctx context.Context) (*grpc.ClientConn, error) { conn, dErr := c.client.DialContext(ctx, "unix", addr) if dErr != nil { return nil, fmt.Errorf( - "connect to machine API socket '%s' through SSH tunnel (is uncloud.service running "+ + "connect to Uncloud API socket '%s' through SSH tunnel (is uncloud.service running "+ "on the remote machine and does the SSH user '%s' have permissions to access the socket?):"+ " %w", addr, c.client.User(), dErr, @@ -94,7 +94,7 @@ func (c *SSHConnector) Connect(ctx context.Context) (*grpc.ClientConn, error) { ), ) if err != nil { - return nil, fmt.Errorf("create machine API client: %w", err) + return nil, fmt.Errorf("create Uncloud API client: %w", err) } return conn, nil } diff --git a/pkg/client/connector/sshcli.go b/pkg/client/connector/sshcli.go index dd47f22a..64877870 100644 --- a/pkg/client/connector/sshcli.go +++ b/pkg/client/connector/sshcli.go @@ -19,7 +19,7 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// SSHCLIConnector establishes a connection to the machine API by executing SSH CLI +// SSHCLIConnector establishes a connection to the Uncloud API by executing SSH CLI // and running `uncloudd dial-stdio` on the remote machine. type SSHCLIConnector struct { config SSHConnectorConfig @@ -104,7 +104,7 @@ func (c *SSHCLIConnector) Connect(ctx context.Context) (*grpc.ClientConn, error) }), ) if err != nil { - return nil, fmt.Errorf("create machine API client: %w", err) + return nil, fmt.Errorf("create Uncloud API client: %w", err) } return grpcConn, nil diff --git a/pkg/client/connector/tcp.go b/pkg/client/connector/tcp.go index f75e23a1..e973dbaf 100644 --- a/pkg/client/connector/tcp.go +++ b/pkg/client/connector/tcp.go @@ -13,7 +13,7 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// TCPConnector establishes a connection to the machine API through a direct TCP connection to an API endpoint. +// TCPConnector establishes a connection to the Uncloud API through a direct TCP connection. type TCPConnector struct { apiAddr netip.AddrPort } @@ -40,7 +40,7 @@ func (c *TCPConnector) Connect(_ context.Context) (*grpc.ClientConn, error) { grpc.WithStreamInterceptor(grpcversion.ClientStreamInterceptor), ) if err != nil { - return nil, fmt.Errorf("create machine API client: %w", err) + return nil, fmt.Errorf("create Uncloud API client: %w", err) } return conn, nil } diff --git a/pkg/client/connector/unix.go b/pkg/client/connector/unix.go index 4246f34f..ed2c58be 100644 --- a/pkg/client/connector/unix.go +++ b/pkg/client/connector/unix.go @@ -10,7 +10,7 @@ import ( "google.golang.org/grpc/credentials/insecure" ) -// UnixConnector establishes a connection to the machine API through a unix domain socket. +// UnixConnector establishes a connection to the Uncloud API through a Unix domain socket. type UnixConnector struct { socketPath string } @@ -31,7 +31,7 @@ func (c *UnixConnector) Connect(_ context.Context) (*grpc.ClientConn, error) { grpc.WithStreamInterceptor(grpcversion.ClientStreamInterceptor), ) if err != nil { - return nil, fmt.Errorf("create machine API client: %w", err) + return nil, fmt.Errorf("create Uncloud API client: %w", err) } return conn, nil } diff --git a/scripts/docker/entrypoint.sh b/scripts/docker/entrypoint.sh index d67309b3..3992ffdc 100755 --- a/scripts/docker/entrypoint.sh +++ b/scripts/docker/entrypoint.sh @@ -35,9 +35,9 @@ echo "Docker in Docker is ready." echo "Loading corrosion image from /images/corrosion.tar..." docker load < /images/corrosion.tar -# Make machine API accessible from the host via port publishing. -echo "Proxying Uncloud API port 51000/tcp to Unix socket /run/uncloud/uncloud.sock..." -socat TCP-LISTEN:51000,reuseaddr,fork,bind="$(hostname -i)" UNIX-CONNECT:/run/uncloud/uncloud.sock & +# Make the Uncloud API accessible from the host via port publishing. +echo "Proxying Uncloud API port 51000/tcp to Unix socket /run/uncloud/api/uncloud.sock..." +socat TCP-LISTEN:51000,reuseaddr,fork,bind="$(hostname -i)" UNIX-CONNECT:/run/uncloud/api/uncloud.sock & # Execute the passed command and wait for it while maintaining signal handling. "$@" & diff --git a/scripts/install.sh b/scripts/install.sh index c8897319..3d6d3451 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -277,11 +277,33 @@ install_uncloud_binaries() { } install_uncloud_systemd() { + local uncloud_socket_path="${INSTALL_SYSTEMD_DIR}/uncloud.socket" local uncloud_service_path="${INSTALL_SYSTEMD_DIR}/uncloud.service" + + mkdir -p "${INSTALL_SYSTEMD_DIR}" + + cat > "${uncloud_socket_path}" << EOF +[Unit] +Description=Uncloud API socket +Before=docker.service + +[Socket] +ListenStream=/run/uncloud/api/uncloud.sock +SocketUser=root +SocketGroup=uncloud +SocketMode=0660 +DirectoryMode=0750 + +[Install] +WantedBy=sockets.target +EOF + log "✓ Systemd unit file created: ${uncloud_socket_path}" + cat > "${uncloud_service_path}" << EOF [Unit] Description=Uncloud machine daemon -After=network-online.target docker.service +Requires=uncloud.socket +After=network-online.target uncloud.socket docker.service Wants=network-online.target [Service] @@ -306,12 +328,11 @@ WantedBy=multi-user.target EOF log "✓ Systemd unit file created: ${uncloud_service_path}" - if [[ "${INSTALL_ONLY}" != "true" ]]; then # Reload systemd to recognize the new or updated unit file. systemctl daemon-reload fi - systemctl enable uncloud.service + systemctl enable uncloud.socket uncloud.service } start_uncloud() { diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index d2d62fba..4213cbdd 100644 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -57,14 +57,17 @@ fi log "⏳ Stopping systemd services..." systemctl stop uncloud.service || log "uncloud.service not running or doesn't exist." +systemctl stop uncloud.socket || log "uncloud.socket not running or doesn't exist." # TODO: remove uncloud-corrosion.service handling in 0.22 once pre-0.20 systemd installs are gone. systemctl stop uncloud-corrosion.service || log "uncloud-corrosion.service not running or doesn't exist." systemctl disable uncloud.service || log "uncloud.service already disabled or doesn't exist." +systemctl disable uncloud.socket || log "uncloud.socket already disabled or doesn't exist." systemctl disable uncloud-corrosion.service || log "uncloud-corrosion.service already disabled or doesn't exist." log "✓ Systemd services stopped." log "⏳ Removing systemd service files..." rm -fv "${INSTALL_SYSTEMD_DIR}/uncloud.service" +rm -fv "${INSTALL_SYSTEMD_DIR}/uncloud.socket" # TODO: remove uncloud-corrosion.service handling in 0.22 once pre-0.20 systemd installs are gone. rm -fv "${INSTALL_SYSTEMD_DIR}/uncloud-corrosion.service" systemctl daemon-reload diff --git a/website/docs/2-getting-started/2-deploy-demo-app.md b/website/docs/2-getting-started/2-deploy-demo-app.md index 06c78eac..3517865c 100644 --- a/website/docs/2-getting-started/2-deploy-demo-app.md +++ b/website/docs/2-getting-started/2-deploy-demo-app.md @@ -142,7 +142,9 @@ WARNING: Access to the remote API on a privileged Docker daemon is equivalent ✓ uncloudd binary installed: /usr/local/bin/uncloudd ⏳ Downloading uninstall script: https://raw.githubusercontent.com/psviderski/uncloud/refs/heads/main/scripts/uninstall.sh ✓ uncloud-uninstall script installed: /usr/local/bin/uncloud-uninstall +✓ Systemd unit file created: /etc/systemd/system/uncloud.socket ✓ Systemd unit file created: /etc/systemd/system/uncloud.service +Created symlink /etc/systemd/system/sockets.target.wants/uncloud.socket → /etc/systemd/system/uncloud.socket. Created symlink /etc/systemd/system/multi-user.target.wants/uncloud.service → /etc/systemd/system/uncloud.service. ⏳ Starting Uncloud machine daemon (uncloud.service)... ✓ Uncloud machine daemon started. @@ -393,9 +395,11 @@ The following actions will be performed: Do you want to proceed with uninstallation? [y/N] y ⏳ Stopping systemd services... Removed /etc/systemd/system/multi-user.target.wants/uncloud.service. +Removed /etc/systemd/system/sockets.target.wants/uncloud.socket. ✓ Systemd services stopped. ⏳ Removing systemd service files... removed '/etc/systemd/system/uncloud.service' +removed '/etc/systemd/system/uncloud.socket' ✓ Systemd service files removed. ⏳ Removing binaries... removed '/usr/local/bin/uncloudd' diff --git a/website/docs/3-concepts/1-clusters/1-connecting.md b/website/docs/3-concepts/1-clusters/1-connecting.md index afb6d1ed..9a06fd7a 100644 --- a/website/docs/3-concepts/1-clusters/1-connecting.md +++ b/website/docs/3-concepts/1-clusters/1-connecting.md @@ -17,9 +17,9 @@ cluster. It has a name and a list of connection details for the machines in that A context is not the same thing as a cluster. It is your local view of a cluster: which machines you can connect through and in what order to try them. Different people or environments may need to reach the same cluster in different ways. -You can also manually create multiple contexts for the same cluster. For example, one that connects through -a machine with a public IP when you're not in the office, and another that connects through a private machine on the -office network when you're on-site to reduce latency. You can switch between them depending on where you are. +You can also manually create multiple contexts for the same cluster. For example, one that connects through a machine +with a public IP when you're not in the office, and another that connects through a private machine on the office +network when you're on-site to reduce latency. You can switch between them depending on where you are. ### Managing contexts @@ -46,11 +46,14 @@ When you run a `uc` command, it determines which cluster to connect to using thi Once the context is resolved, `uc` tries each connection in the context's `connections` list in order until one succeeds. -## User permissions on the machine +## API socket access -When `uc` connects to a machine over SSH, it communicates with the Uncloud daemon through the Unix socket -`/run/uncloud/uncloud.sock` on that machine. The daemon restricts access to the socket to the `root` user and members -of the `uncloud` Linux group. This means your SSH user must be either `root` or a member of the `uncloud` group. +The Uncloud daemon exposes the API through the Unix socket `/run/uncloud/api/uncloud.sock` on each machine. It restricts +access to the `root` user and members of the `uncloud` Linux group. + +### User access + +When `uc` connects to a machine over SSH, the SSH user must be either `root` or a member of the `uncloud` group. In most cases you don't need to set this up manually. When you initialise or add a machine with a non-root user, `uc machine init` and `uc machine add` automatically add that user to the `uncloud` group during installation. @@ -67,6 +70,21 @@ user, close any long-running SSH connections to the machine (for example, SSH Co The same requirement applies when running `uc` locally on a cluster machine with a `unix://` connection. The local user must be `root` or a member of the `uncloud` group. +### Container access + +If a container needs to talk to the Uncloud API over the socket, mount `/run/uncloud/api` as a read-only directory. Do +not mount the socket file directly. The directory mount lets the container see a replacement socket after the daemon +restarts. + +:::warning Full cluster access + +Mounting the Uncloud API socket gives the container the same cluster-wide privileges as a local `uc` client. A process +with access can manage workloads across the cluster and may be able to gain root access to cluster machines through the +workloads it creates. Treat the Uncloud API socket like the Docker socket. Only mount it into containers that you fully +trust. + +::: + ## Global flags and environment variables These flags are available on every `uc` command. They can also be set with an environment variable. The flag takes @@ -95,11 +113,11 @@ uc --connect ssh://root@203.0.113.1 ls # Go's built-in SSH library (no SSH config support, useful when the system ssh is not available) uc --connect ssh+go://root@203.0.113.1 ls -# Direct connection to machine gRPC API over TCP (for advanced users with custom setups) +# Direct connection to the Uncloud API over TCP (for advanced users with custom setups) uc --connect tcp://[fdcc:4439:f545:3ca:5d17:66e5:7c96:40bd]:51000 ls -# Direct connection to machine gRPC API over a Unix socket (for running uc locally on a cluster machine) -uc --connect unix:///run/uncloud/uncloud.sock ls +# Direct connection to the Uncloud API over a Unix socket (for running uc locally on a cluster machine) +uc --connect unix:///run/uncloud/api/uncloud.sock ls ``` :::info diff --git a/website/docs/7-cli-config-reference.md b/website/docs/7-cli-config-reference.md index b84a4fac..6a534344 100644 --- a/website/docs/7-cli-config-reference.md +++ b/website/docs/7-cli-config-reference.md @@ -68,12 +68,12 @@ next one. You can change the default connection with an interactive command Every connection must have exactly one connection type attribute: -| Attribute | Format | Description | -|-----------|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------| -| `ssh` | `user@host[:port]` | Connect using the system `ssh` command with full SSH config support (default for new connections added with `uc machine init/add`) | -| `ssh_go` | `user@host[:port]` | Connect using Go's built-in SSH library (no SSH config support) | -| `tcp` | `host:port` | Connect directly to the machine gRPC API over TCP (for advanced users with custom setups) | -| `unix` | `/run/uncloud/uncloud.sock` | Connect directly to the machine gRPC API over a Unix socket (for running `uc` locally on the cluster machines) | +| Attribute | Format | Description | +|-----------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| `ssh` | `user@host[:port]` | Connect using the system `ssh` command with full SSH config support (default for new connections added with `uc machine init/add`) | +| `ssh_go` | `user@host[:port]` | Connect using Go's built-in SSH library (no SSH config support) | +| `tcp` | `host:port` | Connect directly to the Uncloud API over TCP (for advanced users with custom setups) | +| `unix` | `/run/uncloud/api/uncloud.sock` | Connect directly to the Uncloud API over a Unix socket (for running `uc` locally on a cluster machine) | A connection can also have these optional attributes: