Working creation and removal.

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml

@@ -144,6 +144,7 @@ data:
         </#if>
         - type: raw
           resource: /dev/disk-${ name }
+        <#assign drvCounter = drvCounter + 1/>
         </#list>
         
         display:

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml

@@ -23,16 +23,23 @@ spec:
     #   name: dev-tun
     # - mountPath: /sys/fs/cgroup
     #   name: cgroup
-    - mountPath: /etc/opt/vmrunner
+    - name: config
-      name: config
+      mountPath: /etc/opt/vmrunner
-    - mountPath: /var/local/vm-data
+    - name: vm-data
-      name: vm-data
+      mountPath: /var/local/vm-data
-<#--
     volumeDevices:
-    {{- range $index, $disk := .Values.vm.disks }}
+    <#assign diskCounter = 0/>
-    - devicePath: /dev/disk-{{ $index }}
+    <#list cr.spec.vm.disks.asList() as disk>
-      name: disk-{{ $index }}
+    <#if disk.volumeClaimTemplate.metadata??
-    {{- end }}
+      && disk.volumeClaimTemplate.metadata.name??>
+      <#assign name = disk.volumeClaimTemplate.metadata.name.asString>
+    <#else>
+      <#assign name = "" + diskCounter>
+    </#if>
+    - name: disk-${ name }
+      devicePath: /dev/disk-${ name }
+    <#assign diskCounter = diskCounter + 1/>
+    </#list>
     securityContext:
       privileged: true
   volumes:
@@ -50,16 +57,23 @@ spec:
   #     path: /sys/fs/cgroup
   - name: config
     configMap:
-      name: {{ $.Release.Name }}
+      name: ${ cr.metadata.name.asString }
   - name: vm-data
-    hostPath:
-      path: /var/local/vmrunner/{{ .Release.Name }}
-  {{- range $index, $disk := .Values.vm.disks }}
-  - name: disk-{{ $index }}
     persistentVolumeClaim:
-      claimName: {{ $.Release.Name }}-pvc-{{ $index }}
+      claimName: ${ cr.metadata.name.asString }-data
-  {{- end }}
+  <#assign diskCounter = 0/>
+  <#list cr.spec.vm.disks.asList() as disk>
+  <#if disk.volumeClaimTemplate.metadata??
+    && disk.volumeClaimTemplate.metadata.name??>
+    <#assign name = disk.volumeClaimTemplate.metadata.name.asString>
+  <#else>
+    <#assign name = "" + diskCounter>
+  </#if>
+  - name: disk-${ name }
+    persistentVolumeClaim:
+      claimName: ${ cr.metadata.name.asString }-disk-${ name }
+  <#assign diskCounter = diskCounter + 1/>
+  </#list>
   hostNetwork: true
-  terminationGracePeriodSeconds: 60
+  terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
   restartPolicy: Never
--->

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/vmDataPvc.ftl.yaml

@@ -0,0 +1,16 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  namespace: ${ cr.metadata.namespace.asString }
+  name: ${ cr.metadata.name.asString + "-data" }
+  labels:
+    app.kubernetes.io/name: ${ constants.APP_NAME }
+    app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+    app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
+    
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Mi

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java

@@ -0,0 +1,136 @@
+/*
+ * VM-Operator
+ * Copyright (C) 2023 Michael N. Lipp
+ * 
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.jdrupes.vmoperator.manager;
+
+import io.kubernetes.client.common.KubernetesListObject;
+import io.kubernetes.client.common.KubernetesObject;
+import io.kubernetes.client.custom.V1Patch;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.models.V1ConfigMap;
+import io.kubernetes.client.openapi.models.V1ConfigMapList;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList;
+import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1PodList;
+import io.kubernetes.client.util.generic.GenericKubernetesApi;
+import io.kubernetes.client.util.generic.options.PatchOptions;
+import java.util.Optional;
+
+/**
+ * Helpers for K8s API.
+ */
+@SuppressWarnings({ "PMD.ShortClassName", "PMD.UseUtilityClass" })
+public class K8s {
+
+    /**
+     * Get PVC API.
+     *
+     * @param client the client
+     * @return the generic kubernetes api
+     */
+    public static GenericKubernetesApi<V1PersistentVolumeClaim,
+            V1PersistentVolumeClaimList> pvcApi(ApiClient client) {
+        return new GenericKubernetesApi<>(V1PersistentVolumeClaim.class,
+            V1PersistentVolumeClaimList.class, "", "v1",
+            "persistentvolumeclaims", client);
+    }
+
+    /**
+     * Get config map API.
+     *
+     * @param client the client
+     * @return the generic kubernetes api
+     */
+    public static GenericKubernetesApi<V1ConfigMap,
+            V1ConfigMapList> cmApi(ApiClient client) {
+        return new GenericKubernetesApi<>(V1ConfigMap.class,
+            V1ConfigMapList.class, "", "v1", "configmaps", client);
+    }
+
+    /**
+     * Get pod API.
+     *
+     * @param client the client
+     * @return the generic kubernetes api
+     */
+    public static GenericKubernetesApi<V1Pod, V1PodList>
+            podApi(ApiClient client) {
+        return new GenericKubernetesApi<>(V1Pod.class, V1PodList.class, "",
+            "v1", "pods", client);
+    }
+
+    /**
+     * Get an object from its metadata.
+     *
+     * @param <T> the generic type
+     * @param <LT> the generic type
+     * @param api the api
+     * @param meta the meta
+     * @return the object
+     */
+    public static <T extends KubernetesObject, LT extends KubernetesListObject>
+            Optional<T> get(GenericKubernetesApi<T, LT> api, V1ObjectMeta meta)
+                    throws ApiException {
+        var response = api.get(meta.getNamespace(), meta.getName());
+        if (response.isSuccess()) {
+            return Optional.of(response.getObject());
+        }
+        return Optional.empty();
+    }
+
+    /**
+     * Delete an object.
+     *
+     * @param <T> the generic type
+     * @param <LT> the generic type
+     * @param api the api
+     * @param object the object
+     */
+    public static <T extends KubernetesObject, LT extends KubernetesListObject>
+            void delete(GenericKubernetesApi<T, LT> api, T object)
+                    throws ApiException {
+        api.delete(object.getMetadata().getNamespace(),
+            object.getMetadata().getName()).throwsApiException();
+    }
+
+    /**
+     * Apply the given patch data.
+     *
+     * @param <T> the generic type
+     * @param <LT> the generic type
+     * @param api the api
+     * @param existing the existing
+     * @param update the update
+     * @throws ApiException the api exception
+     */
+    public static <T extends KubernetesObject, LT extends KubernetesListObject>
+            void
+            apply(GenericKubernetesApi<T, LT> api, T existing, String update)
+                    throws ApiException {
+        PatchOptions opts = new PatchOptions();
+        opts.setForce(false);
+        opts.setFieldManager("kubernetes-java-kubectl-apply");
+        api.patch(existing.getMetadata().getNamespace(),
+            existing.getMetadata().getName(), V1Patch.PATCH_FORMAT_APPLY_YAML,
+            new V1Patch(update), opts).throwsApiException();
+    }
+
+}

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java

@@ -29,11 +29,11 @@ import freemarker.template.TemplateException;
 import freemarker.template.TemplateExceptionHandler;
 import freemarker.template.TemplateHashModel;
 import freemarker.template.TemplateNotFoundException;
-import io.kubernetes.client.custom.V1Patch;
 import io.kubernetes.client.openapi.ApiException;
 import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
 import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
-import io.kubernetes.client.util.generic.options.PatchOptions;
+import io.kubernetes.client.util.generic.dynamic.Dynamics;
+import io.kubernetes.client.util.generic.options.ListOptions;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.Collections;
@@ -43,6 +43,7 @@ import static org.jdrupes.vmoperator.manager.Constants.APP_NAME;
 import static org.jdrupes.vmoperator.manager.Constants.VM_OP_GROUP;
 import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME;
 import static org.jdrupes.vmoperator.manager.Constants.VM_OP_VERSION;
+import org.jdrupes.vmoperator.manager.VmDefChanged.Type;
 import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
 import org.jgrapes.core.Channel;
 import org.jgrapes.core.Component;
@@ -90,29 +91,70 @@ public class Reconciler extends Component {
      * @throws KubectlException 
      */
     @Handler
+    @SuppressWarnings("PMD.ConfusingTernary")
     public void onVmDefChanged(VmDefChanged event, WatchChannel channel)
-            throws ApiException, TemplateNotFoundException,
+            throws ApiException, TemplateException, IOException {
-            MalformedTemplateNameException, ParseException, IOException,
-            TemplateException {
         DynamicKubernetesApi vmDefApi = new DynamicKubernetesApi(VM_OP_GROUP,
             VM_OP_VERSION, event.crd().getName(), channel.client());
         var defMeta = event.metadata();
-        var vmDef = vmDefApi.get(defMeta.getNamespace(), defMeta.getName())
+
-            .getObject();
+        // Get common data for all reconciles
+        DynamicKubernetesObject vmDef = null;
+        Map<String, Object> model = null;
+        if (event.type() != Type.DELETED) {
+            vmDef = K8s.get(vmDefApi, defMeta).get();
 
             // Prepare Freemarker model
-        @SuppressWarnings("PMD.UseConcurrentHashMap")
+            model = new HashMap<>();
-        Map<String, Object> model = new HashMap<>();
             model.put("cr", vmDef.getRaw());
             model.put("constants",
                 (TemplateHashModel) new DefaultObjectWrapperBuilder(
                     Configuration.VERSION_2_3_32)
-                    .build().getStaticModels().get(Constants.class.getName()));
+                        .build().getStaticModels()
+                        .get(Constants.class.getName()));
+        }
 
         // Reconcile
+        if (event.type() != Type.DELETED) {
+            reconcileDataPvc(model, channel);
             reconcileDisks(vmDef, channel);
-        reconcileConfigMap(model, channel);
+            reconcileConfigMap(event, model, channel);
-        reconcilePod(model, channel);
+            reconcilePod(event, model, channel);
+        } else {
+            reconcilePod(event, model, channel);
+            reconcileConfigMap(event, model, channel);
+            deletePvcs(event, channel);
+        }
+    }
+
+    @SuppressWarnings("PMD.ConfusingTernary")
+    private void reconcileDataPvc(Map<String, Object> model,
+            WatchChannel channel)
+            throws TemplateException, ApiException, IOException {
+        // Combine template and data and parse result
+        var fmTemplate = fmConfig.getTemplate("vmDataPvc.ftl.yaml");
+        StringWriter out = new StringWriter();
+        fmTemplate.process(model, out);
+        // Avoid Yaml.load due to
+        // https://github.com/kubernetes-client/java/issues/2741
+        var pvcDef = Dynamics.newFromYaml(out.toString());
+
+        // Get API and check if PVC exists
+        DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
+            "persistentvolumeclaims", channel.client());
+        var existing = K8s.get(pvcApi, pvcDef.getMetadata());
+
+        // If PVC does not exist, create. Else patch (apply)
+        if (existing.isEmpty()) {
+            pvcApi.create(pvcDef);
+        } else {
+            // spec is immutable, so mix in existing spec
+            GsonPtr.to(pvcDef.getRaw()).set("spec", GsonPtr
+                .to(existing.get().getRaw()).get(JsonObject.class, "spec")
+                .get().deepCopy());
+            K8s.apply(pvcApi, existing.get(),
+                channel.client().getJSON().serialize(pvcDef));
+        }
     }
 
     private void reconcileDisks(DynamicKubernetesObject vmDef,
@@ -131,22 +173,22 @@ public class Reconciler extends Component {
             int index, JsonObject diskDef, WatchChannel channel)
             throws ApiException {
         var pvcObject = new DynamicKubernetesObject();
-        var pvcDef = GsonPtr.to(pvcObject.getRaw());
+        var pvcRaw = GsonPtr.to(pvcObject.getRaw());
-        var vmDef = GsonPtr.to(vmDefinition.getRaw());
+        var vmRaw = GsonPtr.to(vmDefinition.getRaw());
         var pvcTpl = GsonPtr.to(diskDef).to("volumeClaimTemplate");
 
         // Copy base and metadata from template and add missing/additional data.
         pvcObject.setApiVersion(pvcTpl.getAsString("apiVersion").get());
         pvcObject.setKind(pvcTpl.getAsString("kind").get());
-        var vmName = vmDef.getAsString("metadata", "name").orElse("default");
+        var vmName = vmRaw.getAsString("metadata", "name").orElse("default");
-        pvcDef.get(JsonObject.class).add("metadata",
+        pvcRaw.get(JsonObject.class).add("metadata",
             pvcTpl.to("metadata").get(JsonObject.class).deepCopy());
-        var defMeta = pvcDef.to("metadata");
+        var defMeta = pvcRaw.to("metadata");
         defMeta.computeIfAbsent("namespace", () -> new JsonPrimitive(
-            vmDef.getAsString("metadata", "namespace").orElse("default")));
+            vmRaw.getAsString("metadata", "namespace").orElse("default")));
         defMeta.computeIfAbsent("name", () -> new JsonPrimitive(
             vmName + "-disk-" + index));
-        var pvcLbls = pvcDef.to("metadata", "labels");
+        var pvcLbls = pvcRaw.to("metadata", "labels");
         pvcLbls.set("app.kubernetes.io/name", APP_NAME);
         pvcLbls.set("app.kubernetes.io/instance", vmName);
         pvcLbls.set("app.kubernetes.io/component", "disk");
@@ -155,72 +197,94 @@ public class Reconciler extends Component {
         // Get API and check if PVC exists
         DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
             "persistentvolumeclaims", channel.client());
-        var existing = pvcApi.get(defMeta.getAsString("namespace").get(),
+        var existing = K8s.get(pvcApi, pvcObject.getMetadata());
-            defMeta.getAsString("name").get());
 
         // If PVC does not exist, create. Else patch (apply)
-        if (!existing.isSuccess()) {
+        if (existing.isEmpty()) {
             // PVC does not exist yet, copy spec from template
-            pvcDef.get(JsonObject.class).add("spec",
+            pvcRaw.get(JsonObject.class).add("spec",
                 pvcTpl.to("spec").get(JsonObject.class).deepCopy());
             pvcApi.create(pvcObject);
         } else {
             // spec is immutable, so mix in existing spec
-            pvcDef.set("spec", GsonPtr.to(existing.getObject().getRaw())
+            pvcRaw.set("spec", GsonPtr.to(existing.get().getRaw())
                 .to("spec").get().deepCopy());
-            PatchOptions opts = new PatchOptions();
+            K8s.apply(pvcApi, existing.get(),
-            opts.setForce(false);
+                channel.client().getJSON().serialize(pvcObject));
-            opts.setFieldManager("kubernetes-java-kubectl-apply");
-            pvcApi.patch(pvcObject.getMetadata().getNamespace(),
-                pvcObject.getMetadata().getName(),
-                V1Patch.PATCH_FORMAT_APPLY_YAML,
-                new V1Patch(channel.client().getJSON().serialize(pvcObject)),
-                opts).throwsApiException();
         }
     }
 
-    private void reconcileConfigMap(Map<String, Object> model,
+    private void deletePvcs(VmDefChanged event, WatchChannel channel)
-            WatchChannel channel)
+            throws ApiException {
-            throws TemplateNotFoundException, MalformedTemplateNameException,
+        // Get API and check and list related
-            ParseException, IOException, TemplateException, ApiException {
+        var pvcApi = K8s.pvcApi(channel.client());
+        var pvcs = pvcApi.list(event.metadata().getNamespace(),
+            new ListOptions().labelSelector(
+                "app.kubernetes.io/managed-by=" + VM_OP_NAME
+                    + ",app.kubernetes.io/name=" + APP_NAME
+                    + ",app.kubernetes.io/instance="
+                    + event.metadata().getName()));
+        for (var pvc : pvcs.getObject().getItems()) {
+            K8s.delete(pvcApi, pvc);
+        }
+    }
+
+    private void reconcileConfigMap(VmDefChanged event,
+            Map<String, Object> model, WatchChannel channel)
+            throws IOException, TemplateException, ApiException {
+        // Get API and check if exists
+        DynamicKubernetesApi cmApi = new DynamicKubernetesApi("", "v1",
+            "configmaps", channel.client());
+        var existing = K8s.get(cmApi, event.metadata());
+
+        // If deleted, delete
+        if (event.type() == Type.DELETED) {
+            if (existing.isPresent()) {
+                K8s.delete(cmApi, existing.get());
+            }
+            return;
+        }
+
         // Combine template and data and parse result
         var fmTemplate = fmConfig.getTemplate("runnerConfig.ftl.yaml");
         StringWriter out = new StringWriter();
         fmTemplate.process(model, out);
+        // Avoid Yaml.load due to
+        // https://github.com/kubernetes-client/java/issues/2741
+        var mapDef = Dynamics.newFromYaml(out.toString());
 
         // Apply
-        PatchOptions opts = new PatchOptions();
+        K8s.apply(cmApi, mapDef, out.toString());
-        opts.setForce(false);
+    }
-        opts.setFieldManager("kubernetes-java-kubectl-apply");
+
-        DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
+    private void reconcilePod(VmDefChanged event, Map<String, Object> model,
-            "configmaps", channel.client());
+            WatchChannel channel)
-        var vmDef = GsonPtr.to((JsonObject) model.get("cr"));
+            throws IOException, TemplateException, ApiException {
-        pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(),
+        // Check if exists
-            vmDef.getAsString("metadata", "name").get(),
+        DynamicKubernetesApi podApi = new DynamicKubernetesApi("", "v1",
-            V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()),
+            "pods", channel.client());
-            opts).throwsApiException();
+        var existing = K8s.get(podApi, event.metadata());
+
+        // If deleted, delete
+        if (event.type() == Type.DELETED) {
+            if (existing.isPresent()) {
+                K8s.delete(podApi, existing.get());
+            }
+            return;
         }
 
-    private void reconcilePod(Map<String, Object> model, WatchChannel channel)
-            throws TemplateNotFoundException, MalformedTemplateNameException,
-            ParseException, IOException, TemplateException, ApiException {
         // Combine template and data and parse result
         var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml");
         StringWriter out = new StringWriter();
         fmTemplate.process(model, out);
-        out = null;
+        // Avoid Yaml.load due to
+        // https://github.com/kubernetes-client/java/issues/2741
+        var podDef = Dynamics.newFromYaml(out.toString());
 
-//        // Apply
+        // Nothing can be updated here
-//        PatchOptions opts = new PatchOptions();
+        if (existing.isEmpty()) {
-//        opts.setForce(false);
+            podApi.create(podDef);
-//        opts.setFieldManager("kubernetes-java-kubectl-apply");
+        }
-//        DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
-//            "pod", channel.client());
-//        var vmDef = GsonPtr.to((JsonObject) model.get("cr"));
-//        pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(),
-//            vmDef.getAsString("metadata", "name").get(),
-//            V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()),
-//            opts).throwsApiException();
     }
 
 }