feat: handle internall address differntly if running inside a cluster

.forgejo/workflows/build.yaml

@@ -0,0 +1,21 @@
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: [ nix ]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Nix build
+      run: nix build --extra-experimental-features "flakes nix-command" .#docker-image
+    - name: Setup skopeo
+      run: curl -o /etc/containers/policy.json --create-dirs https://raw.githubusercontent.com/containers/skopeo/refs/heads/main/default-policy.json
+    - name: Push Image
+      run: |
+       skopeo copy --dest-password "$UPLOAD_TOKEN" --dest-username "$USERNAME" "docker-archive://$(realpath result)" "docker://$IMG_URL:${{ github.sha }}" && \
+       skopeo copy --dest-password "$UPLOAD_TOKEN" --dest-username "$USERNAME" "docker-archive://$(realpath result)" "docker://$IMG_URL"
+      env: 
+        USERNAME: tamipes
+        UPLOAD_TOKEN: ${{ secrets.UPLOAD_TOKEN }}
+        IMG_URL: git.tami.moe/tamipes/minecraft-ingress

kube/roles.yaml

@@ -47,7 +47,7 @@ spec:
       serviceAccountName: minecraft-ingress
       containers:
       - name: minecraft-ingress
-        image: git.tami.moe/tamipes/minecraft-ingress:latest
+        image: git.tami.moe/tamipes/minecraft-ingress:testing
         env:
         - name: FILTER_CONN
           value: '(addr == "87.229.85.222") || (addr == "") || (addr == "ogmur.xyz") || (addr == "@mat:matdoes.dev (hi honeypots) ") || (addr == "@mat:matdoes.dev ") || (addr == "slowstack.tv")'

src/kube_cache.rs

@@ -1,6 +1,9 @@
 use std::{collections::HashMap, fmt, sync::Arc, time::Duration};
 
-use k8s_openapi::api::{apps::v1::Deployment, core::v1::Service};
+use k8s_openapi::{
+    api::{apps::v1::Deployment, core::v1::Service},
+    apimachinery::pkg::util::intstr::IntOrString,
+};
 use kube::{
     api::{ListParams, ObjectList, Patch, PatchParams},
     runtime::reflector::Lookup,
@@ -25,13 +28,17 @@ use crate::{
 pub struct KubeCache {
     deployments: Api<Deployment>,
     services: Api<Service>,
+    in_cluster: bool,
 }
 impl KubeCache {
     /// This initializes the creation of a "kubernetes client"
     /// and if it is not possible returns a None.
-    pub fn create() -> Option<KubeCache> {
-        let kubeconfig = kube::config::Kubeconfig::read().unwrap();
-        let client = Client::try_from(kubeconfig).unwrap();
+    pub async fn create() -> Option<KubeCache> {
+        let in_cluster = match std::env::var("KUBERNETES_SERVICE_HOST") {
+            Ok(x) => true,
+            Err(e) => false,
+        };
+        let client = Client::try_default().await.unwrap();
 
         let deployments: Api<Deployment> = Api::default_namespaced(client.clone());
         let services: Api<Service> = Api::default_namespaced(client);
@@ -39,6 +46,7 @@ impl KubeCache {
         return Some(KubeCache {
             deployments,
             services,
+            in_cluster,
         });
     }
     async fn get_dep(&self, name: &str) -> Result<Deployment, kube::Error> {
@@ -64,10 +72,17 @@ impl KubeCache {
         Some(result.name()?.to_string())
     }
 
-    pub async fn query_srv_addr(&self, addr: &str, port: &str) -> Option<String> {
+    pub async fn query_srv(&self, addr: &str, port: &str) -> Option<Service> {
         let deploys = self.get_srvs().await;
-        let result = deploys.iter().find(|x| filter_label_value(x, addr, port))?;
-        Some(result.name()?.to_string())
+        let result = deploys.into_iter().find(|x| {
+            let in_cluster = match x.spec.as_ref().unwrap().type_.as_ref() {
+                Some(t) => t == "ClusterIP",
+                None => false,
+            };
+            let incorrect_type = in_cluster ^ self.in_cluster;
+            !incorrect_type && filter_label_value(&x, addr, port)
+        })?;
+        Some(result)
     }
 
     async fn set_dep_scale(&self, name: &str, num: i32) -> Result<Deployment, kube::Error> {
@@ -101,11 +116,11 @@ impl MinecraftAPI<Server> for McApi {
                 ))
             }
         };
-        let srv_name = match self.cache.query_srv_addr(&addr, &port).await {
+        let service = match self.cache.query_srv(&addr, &port).await {
             Some(x) => x,
             None => {
                 return Err(OpaqueError::create_with_kind(
-                    "Failed to find service name by addr",
+                    "Failed to find service by addr and port",
                     "SrvNameLookupFailed",
                 ))
             }
@@ -117,20 +132,46 @@ impl MinecraftAPI<Server> for McApi {
                 x.to_string()
             )
         })?;
-        let service = self.cache.get_srv(&srv_name).await.map_err(|x| {
-            format!(
-                "Failed to query cache for service with dep_name err:{}",
-                x.to_string()
-            )
-        })?;
         tracing::debug!("found kubernetes deployment & service");
 
+        let service_port_spec = service.clone().spec.unwrap().ports.unwrap();
+        let port = service_port_spec
+            .iter()
+            .find(|x| x.name.clone().unwrap() == "mc-router")
+            .ok_or(OpaqueError::create(
+                "Could not find \"mc-router\" nodePort for server",
+            ))?;
+        let port_string;
+        let inter_addr = match self.cache.in_cluster {
+            false => {
+                let node_port = port
+                    .node_port
+                    .map(|x| x.to_string())
+                    .ok_or(OpaqueError::create("Could not map nodePort to port string"))?;
+                port_string = node_port.clone();
+                format!("localhost:{}", node_port)
+            }
+            true => {
+                let target_port = port.port;
+                let a = format!(
+                    "{}.default.svc.cluster.local:{}",
+                    service
+                        .name()
+                        .ok_or("Could not get name of ClusterIP service")?,
+                    target_port
+                );
+                port_string = a.clone();
+                a
+            }
+        };
+        tracing::info!(inter_addr = inter_addr);
         return Ok(Server {
             dep: deployment,
             srv: service,
             server_addr: addr.to_string(),
-            server_port: port.to_string(),
+            server_port: port_string,
             cache: self.cache.clone(),
+            inter_addr: inter_addr,
         });
     }
 
@@ -209,9 +250,9 @@ impl MinecraftAPI<Server> for McApi {
 }
 
 impl McApi {
-    pub fn create() -> Option<Self> {
+    pub async fn create() -> Option<Self> {
         Some(Self {
-            cache: KubeCache::create()?,
+            cache: KubeCache::create().await?,
             map: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
         })
     }
@@ -224,6 +265,7 @@ pub struct Server {
     server_addr: String,
     server_port: String,
     cache: KubeCache,
+    inter_addr: String,
 }
 impl fmt::Debug for Server {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -309,8 +351,8 @@ impl MinecraftServerHandle for Server {
         }
     }
 
-    fn get_internal_address(&self) -> Option<String> {
-        Some(format!("localhost:{}", self.get_internal_port()?))
+    fn get_internal_address(&self) -> &str {
+        self.inter_addr.as_str()
     }
 
     fn get_addr(&self) -> Option<String> {
@@ -359,11 +401,6 @@ impl MinecraftServerHandle for Server {
         }
     }
 
-    fn get_internal_port(&self) -> Option<String> {
-        let a = self.srv.clone().spec.unwrap().ports.unwrap();
-        let port = a.iter().find(|x| x.name.clone().unwrap() == "mc-router")?;
-        port.node_port.map(|x| x.to_string())
-    }
     fn get_port(&self) -> Option<String> {
         Some(self.server_port.clone())
     }

src/main.rs

@@ -39,7 +39,7 @@ async fn main() {
     let revision: &'static str = env!("COMMIT_HASH");
     tracing::info!(revision);
 
-    let api = kube_cache::McApi::create().unwrap();
+    let api = kube_cache::McApi::create().await.unwrap();
     tracing::info!("initialized kube api");
 
     let config: Config = Default::default();

src/mc_server.rs

@@ -64,16 +64,13 @@ pub trait MinecraftServerHandle: Clone {
     async fn start(&self) -> Result<(), OpaqueError>;
     async fn stop(&self) -> Result<(), OpaqueError>;
     async fn query_status(&self) -> Result<ServerDeploymentStatus, OpaqueError>;
-    fn get_internal_address(&self) -> Option<String>;
-    fn get_internal_port(&self) -> Option<String>;
+    fn get_internal_address(&self) -> &str;
     fn get_addr(&self) -> Option<String>;
     fn get_port(&self) -> Option<String>;
     fn get_motd(&self) -> Option<String>;
 
     async fn query_server_connectable(&self) -> Result<TcpStream, OpaqueError> {
-        let address = self
-            .get_internal_address()
-            .ok_or_else(|| "failed to get internal address from server")?;
+        let address = self.get_internal_address();
         let server_stream = TcpStream::connect(address)
             .await
             .map_err(|_| "failed to connect to minecraft server")?;