From caff2301886b60a353a3f94a0d00e6f361fa9fcf Mon Sep 17 00:00:00 2001 From: ACoolName Date: Sun, 6 Apr 2025 23:52:48 +0300 Subject: [PATCH] added kubernetes - NOT TESTED --- config.yaml | 5 +- factories/instancemanagers.go | 16 +++ go.work.sum | 3 + instancemanager/docker/instance_manager.go | 10 +- instancemanager/instance_manager.go | 8 +- .../kubernetes/instance_manager.go | 121 +++++++++++++++--- instancemanager/kubernetes/kubecon.go | 60 +++++++++ servers/servers.go | 17 +-- 8 files changed, 209 insertions(+), 31 deletions(-) create mode 100644 instancemanager/kubernetes/kubecon.go diff --git a/config.yaml b/config.yaml index 810f394..0b78758 100644 --- a/config.yaml +++ b/config.yaml @@ -38,7 +38,9 @@ authentication: collection: "invite_tokens" instancemanager: - type: "docker" + type: "kubernetes" + kubernetes: + namespace: server-manager docker: browsers_sub_domain: browsers file_browser: @@ -72,4 +74,3 @@ servers_authorization_database: password: "" database: "server_db" collection: "auth_data" - diff --git a/factories/instancemanagers.go b/factories/instancemanagers.go index bb246d2..8556531 100644 --- a/factories/instancemanagers.go +++ b/factories/instancemanagers.go @@ -2,10 +2,12 @@ package factories import ( "errors" + "fmt" "sync" instancemanager "git.acooldomain.co/server-manager/backend/instancemanager" "git.acooldomain.co/server-manager/backend/instancemanager/docker" + "git.acooldomain.co/server-manager/backend/instancemanager/kubernetes" "git.acooldomain.co/server-manager/backend/models" ) @@ -18,6 +20,10 @@ func getDockerCacheKey(config *models.DockerInstanceManagerConfig, siteDomain st return "Docker/" + siteDomain } +func getKubernetesKey(config *models.KubernetesInstanceManagerConfig) string { + return fmt.Sprintf("Kubernetees/%s", config.Namespace) +} + func GetInstanceManager(config models.InstanceManagerConfig, siteDomain string) (instancemanager.InstanceManager, error) { var key string var handler instancemanager.InstanceManager @@ -30,6 +36,11 @@ func GetInstanceManager(config models.InstanceManagerConfig, siteDomain string) return nil, errors.New("missing Docker configuration") } key = getDockerCacheKey(config.Docker, siteDomain) + case models.KUBERNETES: + if config.Kubernetes == nil { + return nil, errors.New("missing Kubernetes configuration") + } + key = getKubernetesKey(config.Kubernetes) default: return nil, errors.New("unsupported database type") } @@ -47,6 +58,11 @@ func GetInstanceManager(config models.InstanceManagerConfig, siteDomain string) if err != nil { return nil, err } + case models.KUBERNETES: + handler, err = kubernetes.NewInstanceManager(*config.Kubernetes) + if err != nil { + return nil, err + } default: return nil, errors.New("unsupported database type") } diff --git a/go.work.sum b/go.work.sum index e5b7675..9e20a58 100644 --- a/go.work.sum +++ b/go.work.sum @@ -172,6 +172,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -416,6 +417,7 @@ github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1D github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/user v0.2.0/go.mod h1:RYstrcWOJpVh+6qzUqp2bU3eaRpdiQeKGlKitaH0PM8= @@ -424,6 +426,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= github.com/nrdcg/auroradns v1.1.0/go.mod h1:O7tViUZbAcnykVnrGkXzIJTHoQCHcgalgAe6X1mzHfk= diff --git a/instancemanager/docker/instance_manager.go b/instancemanager/docker/instance_manager.go index e2f269a..532cd9b 100644 --- a/instancemanager/docker/instance_manager.go +++ b/instancemanager/docker/instance_manager.go @@ -6,7 +6,6 @@ import ( "fmt" "log" "maps" - "net" "strings" instancemanager "git.acooldomain.co/server-manager/backend/instancemanager" @@ -395,7 +394,7 @@ func (im *InstanceManager) DeleteServer(ctx context.Context, serverId string) er // Terminal // Status Changing -func (im *InstanceManager) InteractiveTerminal(ctx context.Context, serverId string) (*net.Conn, error) { +func (im *InstanceManager) InteractiveTerminal(ctx context.Context, serverId string) (*instancemanager.TerminalConnection, error) { server, err := im.GetServer(ctx, serverId) if err != nil { return nil, err @@ -430,7 +429,12 @@ func (im *InstanceManager) InteractiveTerminal(ctx context.Context, serverId str return nil, err } - return &attach.Conn, nil + return &instancemanager.TerminalConnection{ + Conn: attach.Conn, + ResizerFunc: func(width uint, height uint) error { + return im.ResizeTerminal(ctx, serverId, width, height) + }, + }, nil } func (im *InstanceManager) ResizeTerminal(ctx context.Context, serverId string, width uint, height uint) error { diff --git a/instancemanager/instance_manager.go b/instancemanager/instance_manager.go index c59e8cd..6c7a513 100644 --- a/instancemanager/instance_manager.go +++ b/instancemanager/instance_manager.go @@ -16,6 +16,11 @@ type Server struct { Domain string } +type TerminalConnection struct { + Conn net.Conn + ResizerFunc func(width uint, height uint) error +} + type Port struct { Number uint16 Protocol models.PortProtocol @@ -48,8 +53,7 @@ type InstanceManager interface { // Terminal // Status Changing - InteractiveTerminal(ctx context.Context, serverId string) (*net.Conn, error) - ResizeTerminal(ctx context.Context, serverId string, width uint, height uint) error + InteractiveTerminal(ctx context.Context, serverId string) (*TerminalConnection, error) // File Browser diff --git a/instancemanager/kubernetes/instance_manager.go b/instancemanager/kubernetes/instance_manager.go index 3999ca5..cded3b7 100644 --- a/instancemanager/kubernetes/instance_manager.go +++ b/instancemanager/kubernetes/instance_manager.go @@ -2,29 +2,32 @@ package kubernetes import ( "context" - "net" + "io" "strings" + "time" "git.acooldomain.co/server-manager/backend/instancemanager" "git.acooldomain.co/server-manager/backend/models" servermanagerv1 "git.acooldomain.co/server-manager/kubernetes-operator/api/v1alpha1" "github.com/buildkite/shellwords" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/storage/names" names "k8s.io/apiserver/pkg/storage/names" + kubernetesclient "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client" ) type InstanceManager struct { instancemanager.InstanceManager - Config models.KubernetesInstanceManagerConfig - client client.Client + Config models.KubernetesInstanceManagerConfig + client client.Client restCfg *rest.Config - coreV1Cli *kubernetes.Clientset + coreV1Cli *kubernetesclient.Clientset } func convertServerManagerImage(image *servermanagerv1.Image) *instancemanager.Image { @@ -222,32 +225,113 @@ func (i *InstanceManager) DeleteServer(ctx context.Context, serverId string) err // Terminal // Status Changing -func (i *InstanceManager) InteractiveTerminal(ctx context.Context, serverId string) (*net.Conn, error) { - i.client.RESTMapper() - return nil, nil -} +func (i *InstanceManager) InteractiveTerminal(ctx context.Context, serverId string) (*instancemanager.TerminalConnection, error) { + stdinReader, stdinWriter := io.Pipe() + stdoutReader, stdoutWriter := io.Pipe() + resizeChan := make(chan remotecommand.TerminalSize, 1) + queue := &sizeQueue{resizeChan: resizeChan} + req := i.coreV1Cli.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Namespace(i.Config.Namespace). + Name(serverId). + SubResource("attach"). + VersionedParams(&corev1.PodAttachOptions{ + Container: "server", + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, clientgoscheme.ParameterCodec) -func (i *InstanceManager) ResizeTerminal(ctx context.Context, serverId string, width uint, height uint) error { - return nil + executor, err := remotecommand.NewSPDYExecutor(i.restCfg, "POST", req.URL()) + go func() { + defer stdoutWriter.Close() + defer stdinReader.Close() + _ = executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdin: stdinReader, Stdout: stdoutWriter, Stderr: stdoutWriter, Tty: true, TerminalSizeQueue: queue}) + }() + + if err != nil { + return nil, err + } + + k := newKubeCon(stdinWriter, stdoutReader) + + return &instancemanager.TerminalConnection{ + Conn: k, + ResizerFunc: func(width uint, height uint) error { + resizeChan <- remotecommand.TerminalSize{Width: uint16(width), Height: uint16(height)} + return nil + }, + }, nil } // File Browser // Read Only func (i *InstanceManager) GetFileBrowser(ctx context.Context, serverId string) (*models.FileBrowser, error) { - return nil, nil + serverManager := &servermanagerv1.ServerManager{} + err := i.client.Get(ctx, client.ObjectKey{Namespace: i.Config.Namespace, Name: serverId}, serverManager) + if err != nil { + return nil, err + } + + return &models.FileBrowser{ServerId: serverManager.Name, Id: serverManager.Name, Url: serverManager.Status.Browser.Url}, nil } func (i *InstanceManager) ListFileBrowsers(ctx context.Context) ([]models.FileBrowser, error) { - return nil, nil + serverManagers := &servermanagerv1.ServerManagerList{} + err := i.client.List(ctx, serverManagers, &client.ListOptions{Namespace: i.Config.Namespace}) + if err != nil { + return nil, err + } + + fileBrowsers := make([]models.FileBrowser, len(serverManagers.Items)) + for i, serverManager := range serverManagers.Items { + fileBrowsers[i] = models.FileBrowser{ServerId: serverManager.Name, Id: serverManager.Name, Url: serverManager.Status.Browser.Url} + } + + return fileBrowsers, nil } // Status Changing func (i *InstanceManager) StartFileBrowser(ctx context.Context, serverId string) (*models.FileBrowser, error) { - return nil, nil + serverManager := &servermanagerv1.ServerManager{} + err := i.client.Get(ctx, client.ObjectKey{Namespace: i.Config.Namespace, Name: serverId}, serverManager) + if err != nil { + return nil, err + } + + err = i.client.Update(ctx, serverManager) + if err != nil { + return nil, err + } + + childContext, cancelFunc := context.WithTimeout(ctx, time.Second*30) + defer cancelFunc() + + for !serverManager.Status.Browser.Running { + err = i.client.Get(childContext, client.ObjectKey{Name: serverId, Namespace: i.Config.Namespace}, serverManager) + if err != nil { + return nil, err + } + } + + return &models.FileBrowser{Url: serverManager.Status.Browser.Url, Id: serverManager.Name, ServerId: serverManager.Name}, nil } func (i *InstanceManager) StopFileBrowser(ctx context.Context, serverId string) error { + serverManager := &servermanagerv1.ServerManager{} + err := i.client.Get(ctx, client.ObjectKey{Namespace: i.Config.Namespace, Name: serverId}, serverManager) + if err != nil { + return err + } + + err = i.client.Update(ctx, serverManager) + if err != nil { + return err + } + return nil } @@ -266,10 +350,15 @@ func NewInstanceManager(config models.KubernetesInstanceManagerConfig) (*Instanc if err != nil { return nil, err } - + coreV1Cli, err := kubernetesclient.NewForConfig(c) if err != nil { return nil, err } - return &InstanceManager{client: client}, nil + return &InstanceManager{ + client: client, + coreV1Cli: coreV1Cli, + restCfg: c, + Config: config, + }, nil } diff --git a/instancemanager/kubernetes/kubecon.go b/instancemanager/kubernetes/kubecon.go new file mode 100644 index 0000000..81f7cf7 --- /dev/null +++ b/instancemanager/kubernetes/kubecon.go @@ -0,0 +1,60 @@ +package kubernetes + +import ( + "io" + "net" + "time" + + "k8s.io/client-go/tools/remotecommand" +) + +type kubeConn struct { + stdinWriter *io.PipeWriter + stdoutReader *io.PipeReader + closeCh chan struct{} +} + +func (c *kubeConn) Read(b []byte) (int, error) { + return c.stdoutReader.Read(b) +} + +func (c *kubeConn) Write(b []byte) (int, error) { + return c.stdinWriter.Write(b) +} + +func (c *kubeConn) Close() error { + close(c.closeCh) + c.stdinWriter.Close() + return nil +} + +func (c *kubeConn) LocalAddr() net.Addr { return dummyAddr("local") } +func (c *kubeConn) RemoteAddr() net.Addr { return dummyAddr("remote") } +func (c *kubeConn) SetDeadline(t time.Time) error { return nil } +func (c *kubeConn) SetReadDeadline(t time.Time) error { return nil } +func (c *kubeConn) SetWriteDeadline(t time.Time) error { return nil } + +type dummyAddr string + +func (a dummyAddr) Network() string { return string(a) } +func (a dummyAddr) String() string { return string(a) } + +func newKubeCon(stdin *io.PipeWriter, stdout *io.PipeReader) net.Conn { + return &kubeConn{ + stdinWriter: stdin, + stdoutReader: stdout, + closeCh: make(chan struct{}), + } +} + +type sizeQueue struct { + resizeChan chan remotecommand.TerminalSize +} + +func (s *sizeQueue) Next() *remotecommand.TerminalSize { + size, ok := <-s.resizeChan + if !ok { + return nil + } + return &size +} diff --git a/servers/servers.go b/servers/servers.go index f88102b..a4bb522 100644 --- a/servers/servers.go +++ b/servers/servers.go @@ -269,9 +269,9 @@ func (con ServersApi) RunCommand(ctx *gin.Context) { } console := *consolePointer - defer console.Close() + defer console.Conn.Close() - _, err = console.Write([]byte(request.Command + "\n")) + _, err = console.Conn.Write([]byte(request.Command + "\n")) if err != nil { ctx.AbortWithError(500, err) return @@ -289,6 +289,7 @@ func (con ServersApi) AttachServer(ctx *gin.Context) { serverId := ctx.Param("server_id") stop := false var err error + var hijackedPointer *instancemanager.TerminalConnection websocketRead := make(chan Commands) containerRead := make(chan string) @@ -301,13 +302,13 @@ func (con ServersApi) AttachServer(ctx *gin.Context) { close(containerRead) }() - hijackedPointer, err := con.InstanceManager.InteractiveTerminal(ctx, serverId) + hijackedPointer, err = con.InstanceManager.InteractiveTerminal(ctx, serverId) if err != nil { ctx.AbortWithError(500, err) return } hijacked := *hijackedPointer - defer hijacked.Close() + defer hijacked.Conn.Close() ws, err := upgrader.Upgrade(ctx.Writer, ctx.Request, nil) if err != nil { @@ -322,10 +323,10 @@ func (con ServersApi) AttachServer(ctx *gin.Context) { if stop { break } - count, err := hijacked.Read(data) + count, err := hijacked.Conn.Read(data) if err != nil { time.Sleep(500) - hijackedPointer, err := con.InstanceManager.InteractiveTerminal(ctx, serverId) + hijackedPointer, err = con.InstanceManager.InteractiveTerminal(ctx, serverId) if err != nil { ctx.AbortWithError(500, err) stop = true @@ -367,7 +368,7 @@ func (con ServersApi) AttachServer(ctx *gin.Context) { case Command := <-websocketRead: switch Command.CommandType { case "insert": - _, err = hijacked.Write([]byte(Command.Arguments)) + _, err = hijacked.Conn.Write([]byte(Command.Arguments)) if err != nil { log.Printf("Write to docker failed %s", errors.Unwrap(err)) @@ -392,7 +393,7 @@ func (con ServersApi) AttachServer(ctx *gin.Context) { } height := uint(i_height) - err2 = con.InstanceManager.ResizeTerminal(ctx, serverId, width, height) + err2 = hijacked.ResizerFunc(width, height) if err2 != nil { log.Printf("Failed to resize container to %dx%d: %s", width, height, err) }