mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
188 lines
9.2 KiB
Python
188 lines
9.2 KiB
Python
"""Kubernetes operations in one dedicated workload namespace."""
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from kubernetes import client, config, stream
|
|
from kubernetes.client.exceptions import ApiException
|
|
|
|
from errors import Failure
|
|
|
|
|
|
class Cluster:
|
|
identity = "kubernetes"
|
|
|
|
def __init__(self, owner):
|
|
config.load_incluster_config()
|
|
self.api = client.CoreV1Api()
|
|
self.namespace = os.environ["WORKLOAD_NAMESPACE"]
|
|
self.owner = owner
|
|
self.image = os.environ["RUNNER_IMAGE"]
|
|
self.images = json.loads(os.getenv("WORKSPACE_IMAGE_PROFILES", "{}"))
|
|
if not isinstance(self.images, dict) or any(
|
|
not re.fullmatch(r"[a-z][a-z0-9-]{0,40}", k)
|
|
or not isinstance(v, str) or not re.fullmatch(r"[^@\s]+@sha256:[a-f0-9]{64}", v)
|
|
for k, v in self.images.items()):
|
|
raise ValueError("Expected named, digest-pinned workspace images")
|
|
self.mode = os.getenv("DATA_MODE", "ReadWriteMany")
|
|
self.single_mount = self.mode == "ReadWriteOncePod" or os.getenv("ALLOW_SHARED_COMPUTE") == "false"
|
|
self.node = os.getenv("PIN_NODE")
|
|
self.spread = os.getenv("SPREAD_SHARED_DATA", "false") == "true"
|
|
self.storage_class = os.environ["STORAGE_CLASS"]
|
|
if self.mode not in ("ReadWriteMany", "ReadWriteOnce", "ReadWriteOncePod"):
|
|
raise ValueError("Unsupported DATA_MODE")
|
|
if self.mode == "ReadWriteOnce" and not self.node:
|
|
raise ValueError("ReadWriteOnce sharing requires PIN_NODE")
|
|
|
|
def metadata(self, name):
|
|
return {"name": name, "labels": {"ow-workspace-owner": self.owner}}
|
|
|
|
def has_compute(self, name, uid):
|
|
return self.resource("pod", name, uid) is not None
|
|
|
|
def compute(self, workspace, data, profile, previous_uid, data_uid, settings=None):
|
|
return self.pod(workspace, data, profile, previous_uid, settings).metadata.uid
|
|
|
|
def ready(self, workspace, uid):
|
|
pod = self.resource("pod", workspace, uid)
|
|
return pod is not None and any(c.type == "Ready" and c.status == "True" for c in pod.status.conditions or [])
|
|
|
|
def resource(self, kind, name, pinned=None):
|
|
read = self.api.read_namespaced_pod if kind == "pod" else self.api.read_namespaced_persistent_volume_claim
|
|
try:
|
|
obj = read(name, self.namespace, _request_timeout=5)
|
|
except ApiException as e:
|
|
if e.status == 404:
|
|
return None
|
|
raise
|
|
if ((obj.metadata.labels or {}).get("ow-workspace-owner") != self.owner
|
|
or (pinned and obj.metadata.uid != pinned)):
|
|
raise Failure("workspace_identity_changed", 409)
|
|
return obj
|
|
|
|
def ensure(self, kind, body, pinned=None):
|
|
name = body["metadata"]["name"]
|
|
obj = self.resource(kind, name, pinned)
|
|
if obj is None:
|
|
if pinned:
|
|
raise Failure("workspace_data_missing", 409)
|
|
create = self.api.create_namespaced_pod if kind == "pod" else self.api.create_namespaced_persistent_volume_claim
|
|
try:
|
|
create(self.namespace, body, _request_timeout=8)
|
|
except ApiException as e:
|
|
if e.status != 409:
|
|
raise
|
|
obj = self.resource(kind, name)
|
|
if obj is None or obj.metadata.deletion_timestamp:
|
|
raise Failure()
|
|
return obj
|
|
|
|
def claim(self, data, pinned):
|
|
obj = self.ensure("pvc", {
|
|
"metadata": self.metadata(data),
|
|
"spec": {"storageClassName": self.storage_class, "accessModes": [self.mode],
|
|
"resources": {"requests": {"storage": os.getenv("DATA_SIZE", "128Mi")}}},
|
|
}, pinned)
|
|
if obj.spec.storage_class_name != self.storage_class or obj.spec.access_modes != [self.mode]:
|
|
raise Failure("workspace_identity_changed", 409)
|
|
return obj.metadata.uid
|
|
|
|
def pod(self, workspace, data, profile, previous_uid, settings=None):
|
|
settings = settings or {}
|
|
image = settings.get("image", self.image)
|
|
resources = {"small": {"memory": "64Mi", "cpu": "150m"},
|
|
"medium": {"memory": "128Mi", "cpu": "300m"}}[profile]
|
|
spec = {
|
|
"automountServiceAccountToken": False, "enableServiceLinks": False,
|
|
"terminationGracePeriodSeconds": 2, "restartPolicy": "Always",
|
|
"securityContext": {"runAsNonRoot": True, "runAsUser": 65532, "runAsGroup": 65532,
|
|
"fsGroup": 65532, "seccompProfile": {"type": "RuntimeDefault"}},
|
|
"containers": [{
|
|
"name": "workspace", "image": image, "imagePullPolicy": "IfNotPresent",
|
|
"securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True,
|
|
"capabilities": {"drop": ["ALL"]}},
|
|
"resources": {"requests": {"cpu": "10m", "memory": "24Mi"}, "limits": resources},
|
|
"env": [{"name": "WORKSPACE_ID", "value": workspace}],
|
|
"volumeMounts": [{"name": "data", "mountPath": "/data"}],
|
|
}],
|
|
"volumes": [{"name": "data", "persistentVolumeClaim": {"claimName": data}}],
|
|
}
|
|
if "imageProfile" in settings:
|
|
spec["volumes"].append({"name": "tmp", "emptyDir": {}})
|
|
container = spec["containers"][0]
|
|
container["volumeMounts"].append({"name": "tmp", "mountPath": "/tmp"})
|
|
container["env"].append({"name": "OPENWORKER_PROCESS_STATE_ROOT", "value": "/tmp"})
|
|
if self.node:
|
|
spec["nodeSelector"] = {"kubernetes.io/hostname": self.node}
|
|
# Only the provider creates objects with these names. Adopt a creation
|
|
# after a lost response, checking both ownership and the requested binding.
|
|
metadata = self.metadata(workspace)
|
|
metadata["labels"]["ow-workspace-data"] = data
|
|
if settings.get("preview"):
|
|
metadata["labels"]["ow-workspace-preview"] = "enabled"
|
|
if self.spread:
|
|
spec["affinity"] = {"podAntiAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": [{
|
|
"labelSelector": {"matchLabels": {"ow-workspace-data": data}},
|
|
"topologyKey": "kubernetes.io/hostname",
|
|
}]}}
|
|
obj = self.ensure("pod", {"metadata": metadata, "spec": spec})
|
|
mounts = [v.persistent_volume_claim.claim_name for v in obj.spec.volumes if v.persistent_volume_claim]
|
|
if mounts != [data] or obj.spec.containers[0].image != image:
|
|
raise Failure("workspace_identity_changed", 409)
|
|
if previous_uid and obj.metadata.uid != previous_uid:
|
|
raise Failure("workspace_identity_changed", 409)
|
|
return obj
|
|
|
|
def remove(self, kind, name, uid):
|
|
kind = {"compute": "pod", "data": "pvc"}[kind]
|
|
obj = self.resource(kind, name, uid)
|
|
if obj is None:
|
|
return True
|
|
if not obj.metadata.deletion_timestamp:
|
|
delete = self.api.delete_namespaced_pod if kind == "pod" else self.api.delete_namespaced_persistent_volume_claim
|
|
try:
|
|
delete(name, self.namespace, body=client.V1DeleteOptions(
|
|
preconditions=client.V1Preconditions(uid=obj.metadata.uid)), _request_timeout=6)
|
|
except ApiException as e:
|
|
if e.status != 404:
|
|
raise
|
|
return False
|
|
|
|
def process(self, workspace, physical_id, operation):
|
|
if operation.get("name") not in {"process_start", "process_stop", "process_status", "process_logs"}:
|
|
raise Failure("invalid_workspace_request", 400)
|
|
return self.exec_tool(workspace, physical_id, operation, "/app/runner_call.py")
|
|
|
|
def file(self, workspace, physical_id, operation):
|
|
return self.exec_tool(workspace, physical_id, operation, "/app/files.py")
|
|
|
|
def exec_tool(self, workspace, physical_id, operation, helper):
|
|
if self.resource("pod", workspace, physical_id) is None:
|
|
raise Failure("workspace_not_found", 404)
|
|
# Use a separate API client for exec. The streaming transport temporarily
|
|
# replaces the REST transport on the Kubernetes Python client instance.
|
|
with client.ApiClient() as exec_client:
|
|
api = client.CoreV1Api(exec_client)
|
|
ws = stream.stream(api.connect_get_namespaced_pod_exec,
|
|
workspace, self.namespace, container="workspace", command=["python3", helper],
|
|
stdin=True, stdout=True, stderr=True, tty=False, _preload_content=False,
|
|
_request_timeout=8)
|
|
output = ""
|
|
deadline = time.monotonic() + 24
|
|
try:
|
|
ws.write_stdin(json.dumps({"workspace": workspace, "operation": operation}) + "\n")
|
|
while ws.is_open() and time.monotonic() < deadline:
|
|
ws.update(timeout=1)
|
|
output += ws.read_stdout(timeout=0)
|
|
if ws.peek_stderr(timeout=0) or len(output) > 1_048_576:
|
|
raise Failure()
|
|
if ws.is_open() or ws.returncode != 0:
|
|
raise Failure()
|
|
finally:
|
|
ws.close()
|
|
reply = json.loads(output)
|
|
if "error" in reply:
|
|
raise Failure("invalid_workspace_request", 400)
|
|
return reply["result"]
|