mnl/VM-Operator
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 4 Wiki Activity Actions

Deploy pod instead of stateful set

Browse source
This commit is contained in:
Michael Lipp 2024-10-04 15:01:58 +00:00
parent 525696ffe9
commit 83908b7cfd
23 changed files with 762 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

13
buildSrc/.settings/org.eclipse.jdt.core.prefs
View file

@ -1,9 +1,7 @@
#
#Wed Oct 02 14:48:43 CEST 2024
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull
org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=21
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=21
@ -11,12 +9,5 @@ org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error
org.eclipse.jdt.core.compiler.problem.nullReference=warning
org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error
org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=21

2
buildSrc/.settings/org.eclipse.jdt.groovy.core.prefs
View file

@ -1,3 +1,3 @@
eclipse.preferences.version=1
groovy.compiler.level=40
groovy.compiler.level=-1
groovy.script.filters=**/*.dsld,y,**/*.gradle,n

52
buildSrc/build.gradle
View file

@ -1,9 +1,3 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This project uses @Incubating APIs which are subject to change.
*/
plugins {
// Support convention plugins written in Groovy. Convention plugins
// are build scripts in 'src/main' that automatically become available
@ -14,52 +8,24 @@ plugins {
id 'eclipse'
}
repositories {
// Use the plugin portal to apply community plugins in convention plugins.
gradlePluginPortal()
}
sourceSets {
main {
groovy {
srcDirs = ['src']
}
}
test {
groovy {
srcDirs = ['test']
}
}
main {
groovy {
srcDirs = ['src']
}
resources {
srcDirs = ['resources']
}
}
}
eclipse {
project {
file {
// closure executed after .project content is loaded from existing file
// and before gradle build information is merged
beforeMerged { project ->
project.natures.clear()
project.buildCommands.clear()
}
project.natures += 'org.eclipse.buildship.core.gradleprojectnature'
// Don't build, result not used by Eclipse anyway
// project.buildCommand 'org.eclipse.buildship.core.gradleprojectbuilder'
}
}
classpath {
downloadJavadoc = true
downloadSources = true
}
jdt {
file {
withProperties { properties ->
def formatterPrefs = new Properties()
rootProject.file("gradle/org.eclipse.jdt.core.formatter.prefs")
rootProject.file("../gradle/org.eclipse.jdt.core.formatter.prefs")
.withInputStream { formatterPrefs.load(it) }
properties.putAll(formatterPrefs)
}

7
buildSrc/settings.gradle
View file

@ -1,7 +0,0 @@
/*
* This file was generated by the Gradle 'init' task.
*
* This settings file is used to specify which projects to include in your build-logic build.
*/
rootProject.name = 'buildSrc'

2
deploy/vmop-role.yaml
View file

@ -28,9 +28,11 @@ rules:
- apiGroups:
- ""
resources:
- persistentvolumeclaims
- pods
verbs:
- list
- get
- create
- delete
- patch

1
org.jdrupes.vmoperator.common/build.gradle
View file

@ -10,6 +10,7 @@ plugins {
dependencies {
api project(':org.jdrupes.vmoperator.util')
api 'org.jgrapes:org.jgrapes.core:[1.22.1,2)'
api 'io.kubernetes:client-java:[19.0.0,20.0.0)'
api 'org.yaml:snakeyaml'
}

62
org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sObserver.java
View file

@ -30,6 +30,7 @@ import java.time.Instant;
import java.util.function.BiConsumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jgrapes.core.Components;
/**
* An observer that watches namespaced resources in a given context and
@ -73,7 +74,7 @@ public class K8sObserver<O extends KubernetesObject,
*/
@SuppressWarnings({ "PMD.AvoidBranchingStatementAsLastInLoop",
"PMD.UseObjectForClearerAPI", "PMD.AvoidCatchingThrowable",
"PMD.CognitiveComplexity" })
"PMD.CognitiveComplexity", "PMD.AvoidCatchingGenericException" })
public K8sObserver(Class<O> objectClass, Class<L> objectListClass,
K8sClient client, APIResource context, String namespace,
ListOptions options) {
@ -85,38 +86,41 @@ public class K8sObserver<O extends KubernetesObject,
api = new GenericKubernetesApi<>(objectClass, objectListClass,
context.getGroup(), context.getPreferredVersion(),
context.getResourcePlural(), client);
thread = Thread.ofVirtual().unstarted(() -> {
try {
logger.config(() -> "Watching " + context.getResourcePlural()
+ " (" + context.getPreferredVersion() + ")"
+ " in " + namespace);
thread = (Components.useVirtualThreads() ? Thread.ofVirtual()
: Thread.ofPlatform()).unstarted(() -> {
try {
logger
.config(() -> "Watching " + context.getResourcePlural()
+ " (" + context.getPreferredVersion() + ")"
+ " in " + namespace);
// Watch sometimes terminates without apparent reason.
while (!Thread.currentThread().isInterrupted()) {
Instant startedAt = Instant.now();
try {
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
var changed = api.watch(namespace, options).iterator();
while (changed.hasNext()) {
handler.accept(client, changed.next());
// Watch sometimes terminates without apparent reason.
while (!Thread.currentThread().isInterrupted()) {
Instant startedAt = Instant.now();
try {
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
var changed
= api.watch(namespace, options).iterator();
while (changed.hasNext()) {
handler.accept(client, changed.next());
}
} catch (ApiException | RuntimeException e) {
logger.log(Level.FINE, e, () -> "Problem watching"
+ " (will retry): " + e.getMessage());
delayRestart(startedAt);
}
} catch (ApiException | RuntimeException e) {
logger.log(Level.FINE, e, () -> "Problem watching"
+ " (will retry): " + e.getMessage());
delayRestart(startedAt);
}
if (onTerminated != null) {
onTerminated.accept(this, null);
}
} catch (Throwable e) {
logger.log(Level.SEVERE, e, () -> "Probem watching: "
+ e.getMessage());
if (onTerminated != null) {
onTerminated.accept(this, e);
}
}
if (onTerminated != null) {
onTerminated.accept(this, null);
}
} catch (Throwable e) {
logger.log(Level.SEVERE, e, () -> "Probem watching: "
+ e.getMessage());
if (onTerminated != null) {
onTerminated.accept(this, e);
}
}
});
});
}
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")

82
org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sV1PvcStub.java Normal file
View file

@ -0,0 +1,82 @@
/*
* VM-Operator
* Copyright (C) 2024 Michael N. Lipp
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.jdrupes.vmoperator.common;
import io.kubernetes.client.Discovery.APIResource;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList;
import io.kubernetes.client.util.generic.options.ListOptions;
import java.util.Collection;
import java.util.List;
/**
* A stub for pods (v1).
*/
@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
public class K8sV1PvcStub extends
K8sGenericStub<V1PersistentVolumeClaim, V1PersistentVolumeClaimList> {
/** The pods' context. */
public static final APIResource CONTEXT
= new APIResource("", List.of("v1"), "v1", "PersistentVolumeClaim",
true, "persistentvolumeclaims", "persistentvolumeclaim");
/**
* Instantiates a new stub.
*
* @param client the client
* @param namespace the namespace
* @param name the name
*/
protected K8sV1PvcStub(K8sClient client, String namespace, String name) {
super(V1PersistentVolumeClaim.class, V1PersistentVolumeClaimList.class,
client, CONTEXT, namespace, name);
}
/**
* Gets the stub for the given namespace and name.
*
* @param client the client
* @param namespace the namespace
* @param name the name
* @return the kpod stub
*/
public static K8sV1PvcStub get(K8sClient client, String namespace,
String name) {
return new K8sV1PvcStub(client, namespace, name);
}
/**
* Get the stubs for the objects in the given namespace that match
* the criteria from the given options.
*
* @param client the client
* @param namespace the namespace
* @param options the options
* @return the collection
* @throws ApiException the api exception
*/
public static Collection<K8sV1PvcStub> list(K8sClient client,
String namespace, ListOptions options) throws ApiException {
return K8sGenericStub.list(V1PersistentVolumeClaim.class,
V1PersistentVolumeClaimList.class, client, CONTEXT, namespace,
options, (clnt, nscp, name) -> new K8sV1PvcStub(clnt, nscp, name));
}
}

19
org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/VmDefinitionModel.java
View file

@ -37,6 +37,13 @@ import org.jdrupes.vmoperator.util.GsonPtr;
@SuppressWarnings("PMD.DataClass")
public class VmDefinitionModel extends K8sDynamicModel {
/**
* The VM state from the VM definition.
*/
public enum RequestedVmState {
STOPPED, RUNNING
}
/**
* Permissions for accessing and manipulating the VM.
*/
@ -111,6 +118,18 @@ public class VmDefinitionModel extends K8sDynamicModel {
.flatMap(Function.identity()).collect(Collectors.toSet());
}
/**
* Return the requested VM state
*
* @return the string
*/
public RequestedVmState vmState() {
return GsonPtr.to(data()).getAsString("spec", "vm", "state")
.map(s -> "Running".equals(s) ? RequestedVmState.RUNNING
: RequestedVmState.STOPPED)
.orElse(RequestedVmState.STOPPED);
}
/**
* Get the display password serial.
*

1
org.jdrupes.vmoperator.manager.events/build.gradle
View file

@ -9,7 +9,6 @@ plugins {
}
dependencies {
api 'org.jgrapes:org.jgrapes.core:[1.21.0,2)'
api project(':org.jdrupes.vmoperator.common')
api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.16.1,3]'
}

4
org.jdrupes.vmoperator.manager/build.gradle
View file

@ -13,8 +13,8 @@ dependencies {
implementation 'commons-cli:commons-cli:1.5.0'
implementation 'org.jgrapes:org.jgrapes.util:[1.36.0,2)'
implementation 'org.jgrapes:org.jgrapes.io:[2.11.0,3)'
implementation 'org.jgrapes:org.jgrapes.util:[1.38.1,2)'
implementation 'org.jgrapes:org.jgrapes.io:[2.12.1,3)'
implementation 'org.jgrapes:org.jgrapes.http:[3.5.0,4)'
implementation 'org.jgrapes:org.jgrapes.webconsole.base:[1.8.0,2)'

18
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml Normal file
View file

@ -0,0 +1,18 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ runnerDataPvcName }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
spec:
accessModes:
- ReadWriteOnce
<#if reconciler.runnerDataPvc?? && reconciler.runnerDataPvc.storageClassName??>
storageClassName: ${ reconciler.runnerDataPvc.storageClassName }
</#if>
resources:
requests:
storage: 1Mi

16
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml Normal file
View file

@ -0,0 +1,16 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ disk.generatedPvcName }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.annotations??>
annotations:
${ disk.volumeClaimTemplate.metadata.annotations.toString() }
</#if>
spec:
${ disk.volumeClaimTemplate.spec.toString() }

134
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml Normal file
View file

@ -0,0 +1,134 @@
kind: Pod
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/component: ${ constants.APP_NAME }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
# Triggers update of config map mounted in pod
# See https://ahmet.im/blog/kubernetes-secret-volumes-delay/
vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }"
vmoperator.jdrupes.org/version: ${ managerVersion }
ownerReferences:
- apiVersion: ${ cr.apiVersion.asString }
kind: ${ constants.VM_OP_KIND_VM }
name: ${ cr.metadata.name.asString }
uid: ${ cr.metadata.uid.asString }
blockOwnerDeletion: true
controller: false
spec:
containers:
- name: ${ cr.metadata.name.asString }
<#assign image = cr.spec.image>
<#if image.source??>
image: ${ image.source.asString }
<#else>
image: ${ image.repository.asString }/${ image.path.asString }<#if image.version??>:${ image.version.asString }</#if>
</#if>
<#if image.pullPolicy??>
imagePullPolicy: ${ image.pullPolicy.asString }
</#if>
<#if cr.spec.vm.display.spice??>
ports:
<#if cr.spec.vm.display.spice??>
- name: spice
containerPort: ${ cr.spec.vm.display.spice.port.asInt?c }
protocol: TCP
</#if>
</#if>
volumeMounts:
# Not needed because pod is priviledged:
# - mountPath: /dev/kvm
# name: dev-kvm
# - mountPath: /dev/net/tun
# name: dev-tun
# - mountPath: /sys/fs/cgroup
# name: cgroup
- name: config
mountPath: /etc/opt/vmrunner
- name: runner-data
mountPath: /var/local/vm-data
- name: vmop-image-repository
mountPath: ${ constants.IMAGE_REPO_PATH }
volumeDevices:
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate??>
- name: ${ disk.generatedDiskName.asString }
devicePath: /dev/${ disk.generatedDiskName.asString }
</#if>
</#list>
securityContext:
privileged: true
<#if cr.spec.resources??>
resources: ${ cr.spec.resources.toString() }
<#else>
<#if cr.spec.vm.currentCpus?? || cr.spec.vm.currentRam?? >
resources:
requests:
<#if cr.spec.vm.currentCpus?? >
<#assign factor = 2.0 />
<#if reconciler.cpuOvercommit??>
<#assign factor = reconciler.cpuOvercommit * 1.0 />
</#if>
cpu: ${ (parseQuantity(cr.spec.vm.currentCpus.asString) / factor)?c }
</#if>
<#if cr.spec.vm.currentRam?? >
<#assign factor = 1.25 />
<#if reconciler.ramOvercommit??>
<#assign factor = reconciler.ramOvercommit * 1.0 />
</#if>
memory: ${ (parseQuantity(cr.spec.vm.currentRam.asString) / factor)?floor?c }
</#if>
</#if>
</#if>
volumes:
# Not needed because pod is priviledged:
# - name: dev-kvm
# hostPath:
# path: /dev/kvm
# type: CharDevice
# - hostPath:
# path: /dev/net/tun
# type: CharDevice
# name: dev-tun
# - name: cgroup
# hostPath:
# path: /sys/fs/cgroup
- name: config
projected:
sources:
- configMap:
name: ${ cr.metadata.name.asString }
<#if displaySecret??>
- secret:
name: ${ displaySecret }
</#if>
- name: vmop-image-repository
persistentVolumeClaim:
claimName: vmop-image-repository
- name: runner-data
persistentVolumeClaim:
claimName: ${ runnerDataPvcName }
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate??>
- name: ${ disk.generatedDiskName.asString }
persistentVolumeClaim:
claimName: ${ disk.generatedPvcName.asString }
</#if>
</#list>
hostNetwork: true
terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
<#if cr.spec.nodeName??>
nodeName: ${ cr.spec.nodeName.asString }
</#if>
<#if cr.spec.nodeSelector??>
nodeSelector: ${ cr.spec.nodeSelector.toString() }
</#if>
<#if cr.spec.affinity??>
affinity: ${ cr.spec.affinity.toString() }
</#if>
serviceAccountName: vm-runner

4
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java
View file

@ -62,7 +62,6 @@ public abstract class AbstractMonitor<O extends KubernetesObject,
private ListOptions options = new ListOptions();
private final AtomicInteger observerCounter = new AtomicInteger(0);
private ChannelManager<String, C, ?> channelManager;
private boolean channelManagerMaster;
/**
* Initializes the instance.
@ -240,8 +239,7 @@ public abstract class AbstractMonitor<O extends KubernetesObject,
K8s.preferred(context, version), namespace, options)
.handler((c, r) -> {
handleChange(c, r);
if (ResponseType.valueOf(r.type) == ResponseType.DELETED
&& channelManagerMaster) {
if (ResponseType.valueOf(r.type) == ResponseType.DELETED) {
channelManager.remove(r.object.getMetadata().getName());
}
}).onTerminated((o, t) -> {

115
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java Normal file
View file

@ -0,0 +1,115 @@
/*
* VM-Operator
* Copyright (C) 2023 Michael N. Lipp
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.jdrupes.vmoperator.manager;
import freemarker.template.Configuration;
import freemarker.template.TemplateException;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import java.util.logging.Logger;
import org.jdrupes.vmoperator.common.K8sV1PodStub;
import org.jdrupes.vmoperator.common.VmDefinitionModel.RequestedVmState;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
/**
* Delegee for reconciling the pod.
*/
@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
/* default */ class PodReconciler {
protected final Logger logger = Logger.getLogger(getClass().getName());
private final Configuration fmConfig;
/**
* Instantiates a new pod reconciler.
*
* @param fmConfig the fm config
*/
public PodReconciler(Configuration fmConfig) {
this.fmConfig = fmConfig;
}
/**
* Reconcile the pod.
*
* @param event the event
* @param model the model
* @param channel the channel
* @throws IOException Signals that an I/O exception has occurred.
* @throws TemplateException the template exception
* @throws ApiException the api exception
*/
public void reconcile(VmDefChanged event, Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
// Don't do anything if stateful set is still in use (pre v3.4)
if ((Boolean) model.get("usingSts")) {
return;
}
// Get pod stub.
var metadata = event.vmDefinition().getMetadata();
var podStub = K8sV1PodStub.get(channel.client(),
metadata.getNamespace(), metadata.getName());
// Nothing to do if exists and should be running
if (event.vmDefinition().vmState() == RequestedVmState.RUNNING
&& podStub.model().isPresent()) {
return;
}
// Delete if running but should be stopped
if (event.vmDefinition().vmState() == RequestedVmState.STOPPED) {
if (podStub.model().isPresent()) {
podStub.delete();
}
return;
}
// Create pod. First combine template and data and parse result
var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var podDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
// Do apply changes
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
if (podStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(podDef)), opts)
.isEmpty()) {
logger.warning(
() -> "Could not patch pod for " + podStub.name());
}
}
}

197
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java Normal file
View file

@ -0,0 +1,197 @@
/*
* VM-Operator
* Copyright (C) 2023 Michael N. Lipp
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import freemarker.core.ParseException;
import freemarker.template.Configuration;
import freemarker.template.MalformedTemplateNameException;
import freemarker.template.TemplateException;
import freemarker.template.TemplateNotFoundException;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import io.kubernetes.client.util.generic.options.ListOptions;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME;
import org.jdrupes.vmoperator.common.K8sV1PvcStub;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
/**
* Delegee for reconciling the stateful set (effectively the pod).
*/
@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
/* default */ class PvcReconciler {
protected final Logger logger = Logger.getLogger(getClass().getName());
private final Configuration fmConfig;
/**
* Instantiates a new pvc reconciler.
*
* @param fmConfig the fm config
*/
public PvcReconciler(Configuration fmConfig) {
this.fmConfig = fmConfig;
}
/**
* Reconcile the PVCs.
*
* @param event the event
* @param model the model
* @param channel the channel
* @throws IOException Signals that an I/O exception has occurred.
* @throws TemplateException the template exception
* @throws ApiException the api exception
*/
@SuppressWarnings("PMD.AvoidDuplicateLiterals")
public void reconcile(VmDefChanged event, Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
// Existing disks
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector(
"app.kubernetes.io/managed-by=" + VM_OP_NAME + ","
+ "app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=" + metadata.getName());
var knownDisks = K8sV1PvcStub.list(channel.client(),
metadata.getNamespace(), listOpts);
var knownPvcs = knownDisks.stream().map(K8sV1PvcStub::name)
.collect(Collectors.toSet());
// Reconcile runner data pvc
reconcileRunnerDataPvc(event, model, channel, knownPvcs);
// Reconcile pvcs for defined disks
var diskDefs = GsonPtr.to((JsonObject) model.get("cr"))
.getAsListOf(JsonObject.class, "spec", "vm", "disks");
var diskCounter = 0;
for (var diskDef : diskDefs) {
if (!diskDef.has("volumeClaimTemplate")) {
continue;
}
var diskName = GsonPtr.to(diskDef)
.getAsString("volumeClaimTemplate", "metadata", "name")
.map(name -> name + "-disk").orElse("disk-" + diskCounter);
diskCounter += 1;
diskDef.addProperty("generatedDiskName", diskName);
// Don't do anything if pvc with old (sts generated) name exists.
var stsDiskPvcName = diskName + "-" + metadata.getName() + "-0";
if (knownPvcs.contains(stsDiskPvcName)) {
diskDef.addProperty("generatedPvcName", stsDiskPvcName);
continue;
}
// Update PVC
model.put("disk", diskDef);
reconcileRunnerDiskPvc(event, model, channel);
}
model.remove("disk");
}
private void reconcileRunnerDataPvc(VmDefChanged event,
Map<String, Object> model, VmChannel channel,
Set<String> knownPvcs)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
// Look for old (sts generated) name.
var stsRunnerDataPvcName
= "runner-data" + "-" + metadata.getName() + "-0";
if (knownPvcs.contains(stsRunnerDataPvcName)) {
model.put("runnerDataPvcName", stsRunnerDataPvcName);
return;
}
// Generate PVC
model.put("runnerDataPvcName", metadata.getName() + "-runner-data");
var fmTemplate = fmConfig.getTemplate("runnerDataPvc.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var pvcDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
// Do apply changes
var pvcStub = K8sV1PvcStub.get(channel.client(),
metadata.getNamespace(), (String) model.get("runnerDataPvcName"));
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
if (pvcStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(pvcDef)), opts)
.isEmpty()) {
logger.warning(
() -> "Could not patch pvc for " + pvcStub.name());
}
}
private void reconcileRunnerDiskPvc(VmDefChanged event,
Map<String, Object> model, VmChannel channel)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
// Generate PVC
var diskDef = GsonPtr.to((JsonElement) model.get("disk"));
var pvcName = metadata.getName() + "-"
+ diskDef.getAsString("generatedDiskName").get();
diskDef.set("generatedPvcName", pvcName);
var fmTemplate = fmConfig.getTemplate("runnerDiskPvc.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var pvcDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
// Do apply changes
var pvcStub = K8sV1PvcStub.get(channel.client(),
metadata.getNamespace(), GsonPtr.to((JsonElement) model.get("disk"))
.getAsString("generatedPvcName").get());
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
if (pvcStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(pvcDef)), opts)
.isEmpty()) {
logger.warning(
() -> "Could not patch pvc for " + pvcStub.name());
}
}
}

30
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java
View file

@ -69,20 +69,25 @@ import org.jgrapes.util.events.ConfigurationUpdate;
*
* * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/)
* that defines the configuration file for the runner.
*
* * A [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
* that creates
* * the [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/)
* with the Runner instance,
* * a PVC for 1 MiB of persistent storage used by the Runner
* (referred to as the "runnerDataPvc") and
* * the PVCs for the VM's disks.
*
*
* * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
* for 1 MiB of persistent storage used by the Runner (referred to as the
* "runnerDataPvc")
*
* * The PVCs for the VM's disks.
*
* * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the
* runner instance[^oldSts].
*
* * (Optional) A load balancer
* [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)
* that allows the user to access a VM's console without knowing which
* node it runs on.
*
* [^oldSts]: Before version 3.4, the operator created a
* [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
* that created the pod.
*
* The reconciler is part of the {@link Controller} component. It's
* configuration properties are therefore defined in
* ```yaml
@ -135,6 +140,8 @@ public class Reconciler extends Component {
private final ConfigMapReconciler cmReconciler;
private final DisplaySecretReconciler dsReconciler;
private final StatefulSetReconciler stsReconciler;
private final PvcReconciler pvcReconciler;
private final PodReconciler podReconciler;
private final LoadBalancerReconciler lbReconciler;
@SuppressWarnings("PMD.UseConcurrentHashMap")
private final Map<String, Object> config = new HashMap<>();
@ -160,6 +167,8 @@ public class Reconciler extends Component {
cmReconciler = new ConfigMapReconciler(fmConfig);
dsReconciler = new DisplaySecretReconciler();
stsReconciler = new StatefulSetReconciler(fmConfig);
pvcReconciler = new PvcReconciler(fmConfig);
podReconciler = new PodReconciler(fmConfig);
lbReconciler = new LoadBalancerReconciler(fmConfig);
}
@ -206,7 +215,10 @@ public class Reconciler extends Component {
var configMap = cmReconciler.reconcile(model, channel);
model.put("cm", configMap.getRaw());
dsReconciler.reconcile(event, model, channel);
// Manage (eventual) removal of stateful set.
stsReconciler.reconcile(event, model, channel);
pvcReconciler.reconcile(event, model, channel);
podReconciler.reconcile(event, model, channel);
lbReconciler.reconcile(event, model, channel);
}

49
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java
View file

@ -37,7 +37,9 @@ import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
/**
* Delegee for reconciling the stateful set (effectively the pod).
* Before version 3.4, the pod running the VM was created by a stateful set.
* Starting with version 3.4, this reconciler simply deletes the stateful
* set, provided that the VM is not running.
*/
@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
/* default */ class StatefulSetReconciler {
@ -46,7 +48,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
private final Configuration fmConfig;
/**
* Instantiates a new config map reconciler.
* Instantiates a new stateful set reconciler.
*
* @param fmConfig the fm config
*/
@ -64,12 +66,34 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
* @throws TemplateException the template exception
* @throws ApiException the api exception
*/
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
public void reconcile(VmDefChanged event, Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
model.put("usingSts", false);
// Combine template and data and parse result
// If exists, delete when not running or supposed to be not running.
var stsStub = K8sV1StatefulSetStub.get(channel.client(),
metadata.getNamespace(), metadata.getName());
if (stsStub.model().isEmpty()) {
return;
}
// Stateful set still exists, check if replicas is 0 so we can
// delete it.
var stsModel = stsStub.model().get();
if (stsModel.getSpec().getReplicas() == 0) {
stsStub.delete();
return;
}
// Cannot yet delete the stateful set.
model.put("usingSts", true);
// Check if VM is supposed to be stopped. If so,
// set replicas to 0. This is the first step of the transition,
// the stateful set will be deleted when the VM is restarted.
var fmTemplate = fmConfig.getTemplate("runnerSts.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
@ -77,22 +101,13 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
// https://github.com/kubernetes-client/java/issues/2741
var stsDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
// If exists apply changes only when transitioning state
// or not running.
var stsStub = K8sV1StatefulSetStub.get(channel.client(),
metadata.getNamespace(), metadata.getName());
var stsModel = stsStub.model().orElse(null);
if (stsModel != null) {
var current = stsModel.getSpec().getReplicas();
var desired = GsonPtr.to(stsDef.getRaw())
.to("spec").getAsInt("replicas").orElse(1);
if (current == 1 && desired == 1) {
return;
}
var desired = GsonPtr.to(stsDef.getRaw())
.to("spec").getAsInt("replicas").orElse(1);
if (desired == 1) {
return;
}
// Do apply changes
// Do apply changes (set replicas to 0)
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");

25
org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java
View file

@ -2,17 +2,24 @@ package org.jdrupes.vmoperator.manager;
import io.kubernetes.client.Discovery.APIResource;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.options.ListOptions;
import java.io.FileReader;
import java.io.IOException;
import java.util.Map;
import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_GROUP;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_KIND_VM;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME;
import org.jdrupes.vmoperator.common.K8s;
import org.jdrupes.vmoperator.common.K8sClient;
import org.jdrupes.vmoperator.common.K8sDynamicStub;
import org.jdrupes.vmoperator.common.K8sV1ConfigMapStub;
import org.jdrupes.vmoperator.common.K8sV1DeploymentStub;
import org.jdrupes.vmoperator.common.K8sV1StatefulSetStub;
import org.jdrupes.vmoperator.common.K8sV1PodStub;
import org.jdrupes.vmoperator.common.K8sV1PvcStub;
import org.junit.jupiter.api.AfterAll;
import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.BeforeAll;
@ -78,7 +85,7 @@ class BasicTests {
// Wait for created resources
assertTrue(waitForConfigMap(client));
assertTrue(waitForStatefulSet(client));
assertTrue(waitForPvc(client));
// Check config map
var config = K8sV1ConfigMapStub.get(client, "vmop-dev", "unittest-vm")
@ -93,6 +100,15 @@ class BasicTests {
// Cleanup
K8sDynamicStub.get(client, vmsContext, "vmop-dev", "unittest-vm")
.delete();
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector(
"app.kubernetes.io/managed-by=" + VM_OP_NAME + ","
+ "app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=unittest-vm");
var knownPvcs = K8sV1PvcStub.list(client, "vmop-dev", listOpts);
for (var pvc : knownPvcs) {
pvc.delete();
}
}
private boolean waitForConfigMap(K8sClient client)
@ -107,9 +123,10 @@ class BasicTests {
return false;
}
private boolean waitForStatefulSet(K8sClient client)
private boolean waitForPvc(K8sClient client)
throws InterruptedException, ApiException {
var stub = K8sV1StatefulSetStub.get(client, "vmop-dev", "unittest-vm");
var stub
= K8sV1PvcStub.get(client, "vmop-dev", "unittest-vm-runner-data");
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
return true;

6
org.jdrupes.vmoperator.runner.qemu/build.gradle
View file

@ -9,9 +9,9 @@ plugins {
}
dependencies {
implementation 'org.jgrapes:org.jgrapes.core:[1.21.0,2)'
implementation 'org.jgrapes:org.jgrapes.util:[1.36.0,2)'
implementation 'org.jgrapes:org.jgrapes.io:[2.11.0,3)'
implementation 'org.jgrapes:org.jgrapes.core:[1.22.1,2)'
implementation 'org.jgrapes:org.jgrapes.util:[1.38.1,2)'
implementation 'org.jgrapes:org.jgrapes.io:[2.12.1,3)'
implementation 'org.jgrapes:org.jgrapes.http:[3.5.0,4)'
implementation project(':org.jdrupes.vmoperator.common')

59
webpages/vm-operator/controller.md
View file

@ -61,16 +61,20 @@ spec:
## Pod management
The central resource created by the controller is a
[stateful set](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
with the same name as the VM (metadata.name). Its number of replicas is
set to 1 if `spec.vm.state` is "Running" (default is "Stopped" which sets
replicas to 0).
[`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/)
with the same name as the VM (`metadata.name`). The pod is created only
if `spec.vm.state` is "Running" (default is "Stopped" which deletes the
pod)[^oldSts].
Property `spec.guestShutdownStops` (since 2.2.0) controls the effect of a
shutdown initiated by the guest. If set to `false` (default) a new pod
is automatically created by the stateful set controller and the VM thus
restarted. If set to `true`, the runner sets `spec.vm.state` to "Stopped"
before terminating and by this prevents the creation of a new pod.
shutdown initiated by the guest. If set to `false` (default) the pod
and thus the VM is automatically restarted. If set to `true`, the
VM's state is set to "Stopped" when the VM terminates and the pod is
deleted.
[^oldSts]: Before version 3.4, the operator created a
[stateful set](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)
that in turn created the pod and the PVCs (see below).
## Defining the basics
@ -84,7 +88,8 @@ running VMs.
Maybe the most interesting part is the definition of the VM's disks.
This is done by adding one or more `volumeClaimTemplate`s to the
list of disks. As its name suggests, such a template is used by the
controller to generate a PVC.
controller to generate a
[`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/).
The example template does not define any storage. Rather it references
some PV that you must have created first. This may be your first approach
@ -110,24 +115,28 @@ as shown in this example:
The disk will be available as "/dev/*name*-disk" in the VM,
using the string from `.volumeClaimTemplate.metadata.name` as *name*.
If no name is defined in the metadata, then "/dev/disk-*n*"
is used instead, with *n* being the index of the disk
definition in the list of disks.
is used instead, with *n* being the index of the volume claim
template in the list of disks.
Apart from appending "-disk" to the name (or generating the name) the
`volumeClaimTemplate` is simply copied into the stateful set definition
for the VM (with some additional labels, see below). The controller
for stateful sets appends the started pod's name to the name of the
volume claim templates when it creates the PVCs. Therefore you'll
eventually find the PVCs as "*name*-disk-*vmName*-0"
(or "disk-*n*-*vmName*-0").
The name of the generated PVC is the VM's name with "-*name*-disk"
(or the generated name) appended: "*vmName*-*name*-disk"
(or "*vmName*-disk-*n*"). The definition of the PVC is simply a copy
of the information from the `volumeClaimTemplate` (with some additional
labels, see below)[^oldStsDisks].
PVCs generated from stateful set definitions are considered "precious"
and never removed automatically. This behavior fits perfectly for VMs.
Usually, you do not want the disks to be removed automatically when
you (maybe accidentally) remove the CR for the VM. To simplify the lookup
for an eventual (manual) removal, all PVCs are labeled with
"app.kubernetes.io/name: vm-runner", "app.kubernetes.io/instance: *vmName*",
and "app.kubernetes.io/managed-by: vm-operator".
[^oldStsDisks]: Before version 3.4 the `volumeClaimTemplate`s were
copied in the definition of the stateful set. As a stateful set
appends the started pod's name to the name of the volume claim
templates when it creates the PVCs, the PVCs' name were
"*name*-disk-*vmName*-0" (or "disk-*n*-*vmName*-0").
PVCs are never removed automatically. Usually, you do not want your
VMs disks to be removed when you (maybe accidentally) remove the CR
for the VM. To simplify the lookup for an eventual (manual) removal,
all PVCs are labeled with "app.kubernetes.io/name: vm-runner",
"app.kubernetes.io/instance: *vmName*", and
"app.kubernetes.io/managed-by: vm-operator", making it easy to select
the PVCs by label in a delete command.
## Choosing an image for the runner

19
webpages/vm-operator/upgrading.md
View file

@ -5,6 +5,25 @@ layout: vm-operator
# Upgrading
## To version 3.4.0
Starting with this version, the VM-Operator no longer uses a stateful set
with replica set to 1 to (indirectly) start the pod with the VM. Rather
it creates the pod directly. This implies that the PVCs must also be created
by the VM-Operator, which needs additional permissions to do so (update of
`deploy/vmop-role.yaml). As it would be ridiculous to keep the naming scheme
used by the stateful set when generating PVCs, the VM-Operator uses a
[different pattern](controller.html#defining-disks) for creating new PVCs.
The change is backward compatible:
* Running pods created by a stateful set are left alone until stopped.
Only then will the stateful set be removed.
* The VM-Operator looks for existing PVCs generated by a stateful
set in the pre 3.4 versions (naming pattern "*name*-disk-*vmName*-0")
and reuses them. Only new PVCs are generated using the new pattern.
## To version 3.0.0
All configuration files are backward compatible to version 2.3.0.