diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml index 6fcc8c2..ef1b578 100644 --- a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml @@ -144,6 +144,7 @@ data: - type: raw resource: /dev/disk-${ name } + <#assign drvCounter = drvCounter + 1/> display: diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml index 9a56b05..4e27f9b 100644 --- a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml @@ -23,16 +23,23 @@ spec: # name: dev-tun # - mountPath: /sys/fs/cgroup # name: cgroup - - mountPath: /etc/opt/vmrunner - name: config - - mountPath: /var/local/vm-data - name: vm-data -<#-- + - name: config + mountPath: /etc/opt/vmrunner + - name: vm-data + mountPath: /var/local/vm-data volumeDevices: - {{- range $index, $disk := .Values.vm.disks }} - - devicePath: /dev/disk-{{ $index }} - name: disk-{{ $index }} - {{- end }} + <#assign diskCounter = 0/> + <#list cr.spec.vm.disks.asList() as disk> + <#if disk.volumeClaimTemplate.metadata?? + && disk.volumeClaimTemplate.metadata.name??> + <#assign name = disk.volumeClaimTemplate.metadata.name.asString> + <#else> + <#assign name = "" + diskCounter> + + - name: disk-${ name } + devicePath: /dev/disk-${ name } + <#assign diskCounter = diskCounter + 1/> + securityContext: privileged: true volumes: @@ -50,16 +57,23 @@ spec: # path: /sys/fs/cgroup - name: config configMap: - name: {{ $.Release.Name }} + name: ${ cr.metadata.name.asString } - name: vm-data - hostPath: - path: /var/local/vmrunner/{{ .Release.Name }} - {{- range $index, $disk := .Values.vm.disks }} - - name: disk-{{ $index }} persistentVolumeClaim: - claimName: {{ $.Release.Name }}-pvc-{{ $index }} - {{- end }} + claimName: ${ cr.metadata.name.asString }-data + <#assign diskCounter = 0/> + <#list cr.spec.vm.disks.asList() as disk> + <#if disk.volumeClaimTemplate.metadata?? + && disk.volumeClaimTemplate.metadata.name??> + <#assign name = disk.volumeClaimTemplate.metadata.name.asString> + <#else> + <#assign name = "" + diskCounter> + + - name: disk-${ name } + persistentVolumeClaim: + claimName: ${ cr.metadata.name.asString }-disk-${ name } + <#assign diskCounter = diskCounter + 1/> + hostNetwork: true - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c } restartPolicy: Never ---> \ No newline at end of file diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/vmDataPvc.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/vmDataPvc.ftl.yaml new file mode 100644 index 0000000..d33a7f7 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/vmDataPvc.ftl.yaml @@ -0,0 +1,16 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + namespace: ${ cr.metadata.namespace.asString } + name: ${ cr.metadata.name.asString + "-data" } + labels: + app.kubernetes.io/name: ${ constants.APP_NAME } + app.kubernetes.io/instance: ${ cr.metadata.name.asString } + app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME } + +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Mi diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java new file mode 100644 index 0000000..dc4aec0 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java @@ -0,0 +1,136 @@ +/* + * VM-Operator + * Copyright (C) 2023 Michael N. Lipp + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.jdrupes.vmoperator.manager; + +import io.kubernetes.client.common.KubernetesListObject; +import io.kubernetes.client.common.KubernetesObject; +import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapList; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.util.generic.GenericKubernetesApi; +import io.kubernetes.client.util.generic.options.PatchOptions; +import java.util.Optional; + +/** + * Helpers for K8s API. + */ +@SuppressWarnings({ "PMD.ShortClassName", "PMD.UseUtilityClass" }) +public class K8s { + + /** + * Get PVC API. + * + * @param client the client + * @return the generic kubernetes api + */ + public static GenericKubernetesApi pvcApi(ApiClient client) { + return new GenericKubernetesApi<>(V1PersistentVolumeClaim.class, + V1PersistentVolumeClaimList.class, "", "v1", + "persistentvolumeclaims", client); + } + + /** + * Get config map API. + * + * @param client the client + * @return the generic kubernetes api + */ + public static GenericKubernetesApi cmApi(ApiClient client) { + return new GenericKubernetesApi<>(V1ConfigMap.class, + V1ConfigMapList.class, "", "v1", "configmaps", client); + } + + /** + * Get pod API. + * + * @param client the client + * @return the generic kubernetes api + */ + public static GenericKubernetesApi + podApi(ApiClient client) { + return new GenericKubernetesApi<>(V1Pod.class, V1PodList.class, "", + "v1", "pods", client); + } + + /** + * Get an object from its metadata. + * + * @param the generic type + * @param the generic type + * @param api the api + * @param meta the meta + * @return the object + */ + public static + Optional get(GenericKubernetesApi api, V1ObjectMeta meta) + throws ApiException { + var response = api.get(meta.getNamespace(), meta.getName()); + if (response.isSuccess()) { + return Optional.of(response.getObject()); + } + return Optional.empty(); + } + + /** + * Delete an object. + * + * @param the generic type + * @param the generic type + * @param api the api + * @param object the object + */ + public static + void delete(GenericKubernetesApi api, T object) + throws ApiException { + api.delete(object.getMetadata().getNamespace(), + object.getMetadata().getName()).throwsApiException(); + } + + /** + * Apply the given patch data. + * + * @param the generic type + * @param the generic type + * @param api the api + * @param existing the existing + * @param update the update + * @throws ApiException the api exception + */ + public static + void + apply(GenericKubernetesApi api, T existing, String update) + throws ApiException { + PatchOptions opts = new PatchOptions(); + opts.setForce(false); + opts.setFieldManager("kubernetes-java-kubectl-apply"); + api.patch(existing.getMetadata().getNamespace(), + existing.getMetadata().getName(), V1Patch.PATCH_FORMAT_APPLY_YAML, + new V1Patch(update), opts).throwsApiException(); + } + +} diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java index 51d3167..b038442 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java @@ -29,11 +29,11 @@ import freemarker.template.TemplateException; import freemarker.template.TemplateExceptionHandler; import freemarker.template.TemplateHashModel; import freemarker.template.TemplateNotFoundException; -import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi; import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject; -import io.kubernetes.client.util.generic.options.PatchOptions; +import io.kubernetes.client.util.generic.dynamic.Dynamics; +import io.kubernetes.client.util.generic.options.ListOptions; import java.io.IOException; import java.io.StringWriter; import java.util.Collections; @@ -43,6 +43,7 @@ import static org.jdrupes.vmoperator.manager.Constants.APP_NAME; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_GROUP; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_VERSION; +import org.jdrupes.vmoperator.manager.VmDefChanged.Type; import org.jdrupes.vmoperator.util.ExtendedObjectWrapper; import org.jgrapes.core.Channel; import org.jgrapes.core.Component; @@ -90,29 +91,70 @@ public class Reconciler extends Component { * @throws KubectlException */ @Handler + @SuppressWarnings("PMD.ConfusingTernary") public void onVmDefChanged(VmDefChanged event, WatchChannel channel) - throws ApiException, TemplateNotFoundException, - MalformedTemplateNameException, ParseException, IOException, - TemplateException { + throws ApiException, TemplateException, IOException { DynamicKubernetesApi vmDefApi = new DynamicKubernetesApi(VM_OP_GROUP, VM_OP_VERSION, event.crd().getName(), channel.client()); var defMeta = event.metadata(); - var vmDef = vmDefApi.get(defMeta.getNamespace(), defMeta.getName()) - .getObject(); - // Prepare Freemarker model - @SuppressWarnings("PMD.UseConcurrentHashMap") - Map model = new HashMap<>(); - model.put("cr", vmDef.getRaw()); - model.put("constants", - (TemplateHashModel) new DefaultObjectWrapperBuilder( - Configuration.VERSION_2_3_32) - .build().getStaticModels().get(Constants.class.getName())); + // Get common data for all reconciles + DynamicKubernetesObject vmDef = null; + Map model = null; + if (event.type() != Type.DELETED) { + vmDef = K8s.get(vmDefApi, defMeta).get(); + + // Prepare Freemarker model + model = new HashMap<>(); + model.put("cr", vmDef.getRaw()); + model.put("constants", + (TemplateHashModel) new DefaultObjectWrapperBuilder( + Configuration.VERSION_2_3_32) + .build().getStaticModels() + .get(Constants.class.getName())); + } // Reconcile - reconcileDisks(vmDef, channel); - reconcileConfigMap(model, channel); - reconcilePod(model, channel); + if (event.type() != Type.DELETED) { + reconcileDataPvc(model, channel); + reconcileDisks(vmDef, channel); + reconcileConfigMap(event, model, channel); + reconcilePod(event, model, channel); + } else { + reconcilePod(event, model, channel); + reconcileConfigMap(event, model, channel); + deletePvcs(event, channel); + } + } + + @SuppressWarnings("PMD.ConfusingTernary") + private void reconcileDataPvc(Map model, + WatchChannel channel) + throws TemplateException, ApiException, IOException { + // Combine template and data and parse result + var fmTemplate = fmConfig.getTemplate("vmDataPvc.ftl.yaml"); + StringWriter out = new StringWriter(); + fmTemplate.process(model, out); + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var pvcDef = Dynamics.newFromYaml(out.toString()); + + // Get API and check if PVC exists + DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1", + "persistentvolumeclaims", channel.client()); + var existing = K8s.get(pvcApi, pvcDef.getMetadata()); + + // If PVC does not exist, create. Else patch (apply) + if (existing.isEmpty()) { + pvcApi.create(pvcDef); + } else { + // spec is immutable, so mix in existing spec + GsonPtr.to(pvcDef.getRaw()).set("spec", GsonPtr + .to(existing.get().getRaw()).get(JsonObject.class, "spec") + .get().deepCopy()); + K8s.apply(pvcApi, existing.get(), + channel.client().getJSON().serialize(pvcDef)); + } } private void reconcileDisks(DynamicKubernetesObject vmDef, @@ -131,22 +173,22 @@ public class Reconciler extends Component { int index, JsonObject diskDef, WatchChannel channel) throws ApiException { var pvcObject = new DynamicKubernetesObject(); - var pvcDef = GsonPtr.to(pvcObject.getRaw()); - var vmDef = GsonPtr.to(vmDefinition.getRaw()); + var pvcRaw = GsonPtr.to(pvcObject.getRaw()); + var vmRaw = GsonPtr.to(vmDefinition.getRaw()); var pvcTpl = GsonPtr.to(diskDef).to("volumeClaimTemplate"); // Copy base and metadata from template and add missing/additional data. pvcObject.setApiVersion(pvcTpl.getAsString("apiVersion").get()); pvcObject.setKind(pvcTpl.getAsString("kind").get()); - var vmName = vmDef.getAsString("metadata", "name").orElse("default"); - pvcDef.get(JsonObject.class).add("metadata", + var vmName = vmRaw.getAsString("metadata", "name").orElse("default"); + pvcRaw.get(JsonObject.class).add("metadata", pvcTpl.to("metadata").get(JsonObject.class).deepCopy()); - var defMeta = pvcDef.to("metadata"); + var defMeta = pvcRaw.to("metadata"); defMeta.computeIfAbsent("namespace", () -> new JsonPrimitive( - vmDef.getAsString("metadata", "namespace").orElse("default"))); + vmRaw.getAsString("metadata", "namespace").orElse("default"))); defMeta.computeIfAbsent("name", () -> new JsonPrimitive( vmName + "-disk-" + index)); - var pvcLbls = pvcDef.to("metadata", "labels"); + var pvcLbls = pvcRaw.to("metadata", "labels"); pvcLbls.set("app.kubernetes.io/name", APP_NAME); pvcLbls.set("app.kubernetes.io/instance", vmName); pvcLbls.set("app.kubernetes.io/component", "disk"); @@ -155,72 +197,94 @@ public class Reconciler extends Component { // Get API and check if PVC exists DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1", "persistentvolumeclaims", channel.client()); - var existing = pvcApi.get(defMeta.getAsString("namespace").get(), - defMeta.getAsString("name").get()); + var existing = K8s.get(pvcApi, pvcObject.getMetadata()); // If PVC does not exist, create. Else patch (apply) - if (!existing.isSuccess()) { + if (existing.isEmpty()) { // PVC does not exist yet, copy spec from template - pvcDef.get(JsonObject.class).add("spec", + pvcRaw.get(JsonObject.class).add("spec", pvcTpl.to("spec").get(JsonObject.class).deepCopy()); pvcApi.create(pvcObject); } else { // spec is immutable, so mix in existing spec - pvcDef.set("spec", GsonPtr.to(existing.getObject().getRaw()) + pvcRaw.set("spec", GsonPtr.to(existing.get().getRaw()) .to("spec").get().deepCopy()); - PatchOptions opts = new PatchOptions(); - opts.setForce(false); - opts.setFieldManager("kubernetes-java-kubectl-apply"); - pvcApi.patch(pvcObject.getMetadata().getNamespace(), - pvcObject.getMetadata().getName(), - V1Patch.PATCH_FORMAT_APPLY_YAML, - new V1Patch(channel.client().getJSON().serialize(pvcObject)), - opts).throwsApiException(); + K8s.apply(pvcApi, existing.get(), + channel.client().getJSON().serialize(pvcObject)); } } - private void reconcileConfigMap(Map model, - WatchChannel channel) - throws TemplateNotFoundException, MalformedTemplateNameException, - ParseException, IOException, TemplateException, ApiException { + private void deletePvcs(VmDefChanged event, WatchChannel channel) + throws ApiException { + // Get API and check and list related + var pvcApi = K8s.pvcApi(channel.client()); + var pvcs = pvcApi.list(event.metadata().getNamespace(), + new ListOptions().labelSelector( + "app.kubernetes.io/managed-by=" + VM_OP_NAME + + ",app.kubernetes.io/name=" + APP_NAME + + ",app.kubernetes.io/instance=" + + event.metadata().getName())); + for (var pvc : pvcs.getObject().getItems()) { + K8s.delete(pvcApi, pvc); + } + } + + private void reconcileConfigMap(VmDefChanged event, + Map model, WatchChannel channel) + throws IOException, TemplateException, ApiException { + // Get API and check if exists + DynamicKubernetesApi cmApi = new DynamicKubernetesApi("", "v1", + "configmaps", channel.client()); + var existing = K8s.get(cmApi, event.metadata()); + + // If deleted, delete + if (event.type() == Type.DELETED) { + if (existing.isPresent()) { + K8s.delete(cmApi, existing.get()); + } + return; + } + // Combine template and data and parse result var fmTemplate = fmConfig.getTemplate("runnerConfig.ftl.yaml"); StringWriter out = new StringWriter(); fmTemplate.process(model, out); + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var mapDef = Dynamics.newFromYaml(out.toString()); // Apply - PatchOptions opts = new PatchOptions(); - opts.setForce(false); - opts.setFieldManager("kubernetes-java-kubectl-apply"); - DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1", - "configmaps", channel.client()); - var vmDef = GsonPtr.to((JsonObject) model.get("cr")); - pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(), - vmDef.getAsString("metadata", "name").get(), - V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()), - opts).throwsApiException(); + K8s.apply(cmApi, mapDef, out.toString()); } - private void reconcilePod(Map model, WatchChannel channel) - throws TemplateNotFoundException, MalformedTemplateNameException, - ParseException, IOException, TemplateException, ApiException { + private void reconcilePod(VmDefChanged event, Map model, + WatchChannel channel) + throws IOException, TemplateException, ApiException { + // Check if exists + DynamicKubernetesApi podApi = new DynamicKubernetesApi("", "v1", + "pods", channel.client()); + var existing = K8s.get(podApi, event.metadata()); + + // If deleted, delete + if (event.type() == Type.DELETED) { + if (existing.isPresent()) { + K8s.delete(podApi, existing.get()); + } + return; + } + // Combine template and data and parse result var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml"); StringWriter out = new StringWriter(); fmTemplate.process(model, out); - out = null; + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var podDef = Dynamics.newFromYaml(out.toString()); -// // Apply -// PatchOptions opts = new PatchOptions(); -// opts.setForce(false); -// opts.setFieldManager("kubernetes-java-kubectl-apply"); -// DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1", -// "pod", channel.client()); -// var vmDef = GsonPtr.to((JsonObject) model.get("cr")); -// pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(), -// vmDef.getAsString("metadata", "name").get(), -// V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()), -// opts).throwsApiException(); + // Nothing can be updated here + if (existing.isEmpty()) { + podApi.create(podDef); + } } }