Handle conflict properly.
This commit is contained in:
parent
21108771d9
commit
d5e589709f
3 changed files with 100 additions and 82 deletions
|
|
@ -193,42 +193,41 @@ public class K8sGenericStub<O extends KubernetesObject,
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Updates the object's status, retrying for the given number of times
|
* Updates the object's status.
|
||||||
* if the update fails due to a conflict.
|
|
||||||
*
|
*
|
||||||
* @param object the current state of the object (passed to `status`)
|
* @param object the current state of the object (passed to `status`)
|
||||||
* @param status function that returns the new status
|
* @param status function that returns the new status
|
||||||
* @param retries the retries
|
* @return the updated model or empty if the object was not found
|
||||||
* @return the updated model or empty if not successful
|
|
||||||
* @throws ApiException the api exception
|
* @throws ApiException the api exception
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("PMD.AssignmentInOperand")
|
@SuppressWarnings("PMD.AssignmentInOperand")
|
||||||
public Optional<O> updateStatus(O object,
|
public Optional<O> updateStatus(O object, Function<O, Object> status)
|
||||||
Function<O, Object> status, int retries) throws ApiException {
|
throws ApiException {
|
||||||
while (true) {
|
|
||||||
try {
|
|
||||||
return K8s.optional(api.updateStatus(object, status));
|
return K8s.optional(api.updateStatus(object, status));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the object and updates the status. In case of conflict, retries
|
||||||
|
* up to `retries` times.
|
||||||
|
*
|
||||||
|
* @param status the status
|
||||||
|
* @param retries the retries in case of conflict
|
||||||
|
* @return the updated model or empty if the object was not found
|
||||||
|
* @throws ApiException the api exception
|
||||||
|
*/
|
||||||
|
@SuppressWarnings({ "PMD.AssignmentInOperand", "PMD.UnusedAssignment" })
|
||||||
|
public Optional<O> updateStatus(Function<O, Object> status, int retries)
|
||||||
|
throws ApiException {
|
||||||
|
try {
|
||||||
|
return updateStatus(api.get(namespace, name).throwsApiException()
|
||||||
|
.getObject(), status);
|
||||||
} catch (ApiException e) {
|
} catch (ApiException e) {
|
||||||
if (HttpURLConnection.HTTP_CONFLICT != e.getCode()
|
if (HttpURLConnection.HTTP_CONFLICT != e.getCode()
|
||||||
|| retries-- <= 0) {
|
|| retries-- <= 0) {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
return Optional.empty();
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Updates the object's status, retrying up to 16 times if there
|
|
||||||
* is a conflict.
|
|
||||||
*
|
|
||||||
* @param object the current state of the object (passed to `status`)
|
|
||||||
* @param status function that returns the new status
|
|
||||||
* @return the updated model or empty if not successful
|
|
||||||
* @throws ApiException the api exception
|
|
||||||
*/
|
|
||||||
public Optional<O> updateStatus(O object,
|
|
||||||
Function<O, Object> status) throws ApiException {
|
|
||||||
return updateStatus(object, status, 16);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -241,8 +240,7 @@ public class K8sGenericStub<O extends KubernetesObject,
|
||||||
*/
|
*/
|
||||||
public Optional<O> updateStatus(Function<O, Object> status)
|
public Optional<O> updateStatus(Function<O, Object> status)
|
||||||
throws ApiException {
|
throws ApiException {
|
||||||
return updateStatus(
|
return updateStatus(status, 16);
|
||||||
api.get(namespace, name).throwsApiException().getObject(), status);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ import io.kubernetes.client.custom.V1Patch;
|
||||||
import io.kubernetes.client.openapi.ApiException;
|
import io.kubernetes.client.openapi.ApiException;
|
||||||
import io.kubernetes.client.openapi.Configuration;
|
import io.kubernetes.client.openapi.Configuration;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
|
|
@ -211,7 +212,10 @@ public class Controller extends Component {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update the assignment information in the status of the VM CR.
|
* Attempt to Update the assignment information in the status of the
|
||||||
|
* VM CR. Returns true if successful. The handler does not attempt
|
||||||
|
* retries, because in case of failure it will be necessary to
|
||||||
|
* re-evaluate the chosen VM.
|
||||||
*
|
*
|
||||||
* @param event the event
|
* @param event the event
|
||||||
* @param channel the channel
|
* @param channel the channel
|
||||||
|
|
@ -220,18 +224,27 @@ public class Controller extends Component {
|
||||||
@Handler
|
@Handler
|
||||||
public void onUpdatedAssignment(UpdateAssignment event, VmChannel channel)
|
public void onUpdatedAssignment(UpdateAssignment event, VmChannel channel)
|
||||||
throws ApiException {
|
throws ApiException {
|
||||||
|
try {
|
||||||
var vmDef = channel.vmDefinition();
|
var vmDef = channel.vmDefinition();
|
||||||
var vmStub = VmDefinitionStub.get(channel.client(),
|
var vmStub = VmDefinitionStub.get(channel.client(),
|
||||||
new GroupVersionKind(VM_OP_GROUP, "", VM_OP_KIND_VM),
|
new GroupVersionKind(VM_OP_GROUP, "", VM_OP_KIND_VM),
|
||||||
vmDef.namespace(), vmDef.name());
|
vmDef.namespace(), vmDef.name());
|
||||||
vmStub.updateStatus(from -> {
|
if (vmStub.updateStatus(vmDef, from -> {
|
||||||
JsonObject status = from.statusJson();
|
JsonObject status = from.statusJson();
|
||||||
var assignment = GsonPtr.to(status).to("assignment");
|
var assignment = GsonPtr.to(status).to("assignment");
|
||||||
assignment.set("pool", event.usedPool());
|
assignment.set("pool", event.usedPool());
|
||||||
assignment.set("user", event.toUser());
|
assignment.set("user", event.toUser());
|
||||||
assignment.set("lastUsed", Instant.now().toString());
|
assignment.set("lastUsed", Instant.now().toString());
|
||||||
return status;
|
return status;
|
||||||
});
|
}).isPresent()) {
|
||||||
event.setResult(true);
|
event.setResult(true);
|
||||||
}
|
}
|
||||||
|
} catch (ApiException e) {
|
||||||
|
// Log exceptions except for conflict, which can be expected
|
||||||
|
if (HttpURLConnection.HTTP_CONFLICT != e.getCode()) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
event.setResult(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -256,20 +256,21 @@ public class VmMonitor extends
|
||||||
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
|
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
|
||||||
public void onAssignVm(AssignVm event)
|
public void onAssignVm(AssignVm event)
|
||||||
throws ApiException, InterruptedException {
|
throws ApiException, InterruptedException {
|
||||||
|
while (true) {
|
||||||
// Search for existing assignment.
|
// Search for existing assignment.
|
||||||
var assignedVm = channelManager.channels().stream()
|
var vmQuery = channelManager.channels().stream()
|
||||||
.filter(c -> c.vmDefinition().assignedFrom()
|
.filter(c -> c.vmDefinition().assignedFrom()
|
||||||
.map(p -> p.equals(event.fromPool())).orElse(false))
|
.map(p -> p.equals(event.fromPool())).orElse(false))
|
||||||
.filter(c -> c.vmDefinition().assignedTo()
|
.filter(c -> c.vmDefinition().assignedTo()
|
||||||
.map(u -> u.equals(event.toUser())).orElse(false))
|
.map(u -> u.equals(event.toUser())).orElse(false))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
if (assignedVm.isPresent()) {
|
if (vmQuery.isPresent()) {
|
||||||
var vmDef = assignedVm.get().vmDefinition();
|
var vmDef = vmQuery.get().vmDefinition();
|
||||||
event.setResult(new VmData(vmDef, assignedVm.get()));
|
event.setResult(new VmData(vmDef, vmQuery.get()));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the pool definition assignability check
|
// Get the pool definition for checking possible assignment
|
||||||
VmPool vmPool = newEventPipeline().fire(new GetPools()
|
VmPool vmPool = newEventPipeline().fire(new GetPools()
|
||||||
.withName(event.fromPool())).get().stream().findFirst()
|
.withName(event.fromPool())).get().stream().findFirst()
|
||||||
.orElse(null);
|
.orElse(null);
|
||||||
|
|
@ -278,7 +279,7 @@ public class VmMonitor extends
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find available VM.
|
// Find available VM.
|
||||||
assignedVm = channelManager.channels().stream()
|
vmQuery = channelManager.channels().stream()
|
||||||
.filter(c -> vmPool.isAssignable(c.vmDefinition()))
|
.filter(c -> vmPool.isAssignable(c.vmDefinition()))
|
||||||
.sorted(Comparator.comparing((VmChannel c) -> c.vmDefinition()
|
.sorted(Comparator.comparing((VmChannel c) -> c.vmDefinition()
|
||||||
.assignmentLastUsed().orElse(Instant.ofEpochSecond(0)))
|
.assignmentLastUsed().orElse(Instant.ofEpochSecond(0)))
|
||||||
|
|
@ -286,19 +287,25 @@ public class VmMonitor extends
|
||||||
.findFirst();
|
.findFirst();
|
||||||
|
|
||||||
// None found
|
// None found
|
||||||
if (assignedVm.isEmpty()) {
|
if (vmQuery.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assign to user
|
// Assign to user
|
||||||
assignedVm.get().pipeline().fire(new UpdateAssignment(vmPool.name(),
|
var chosenVm = vmQuery.get();
|
||||||
event.toUser()), assignedVm.get()).get();
|
var vmPipeline = chosenVm.pipeline();
|
||||||
var vmDef = assignedVm.get().vmDefinition();
|
if (Optional.ofNullable(vmPipeline.fire(new UpdateAssignment(
|
||||||
event.setResult(new VmData(vmDef, assignedVm.get()));
|
vmPool.name(), event.toUser()), chosenVm).get())
|
||||||
|
.orElse(false)) {
|
||||||
|
var vmDef = chosenVm.vmDefinition();
|
||||||
|
event.setResult(new VmData(vmDef, chosenVm));
|
||||||
|
|
||||||
// Make sure that a newly assigned VM is running.
|
// Make sure that a newly assigned VM is running.
|
||||||
assignedVm.get().pipeline().fire(new ModifyVm(vmDef.name(),
|
chosenVm.pipeline().fire(new ModifyVm(vmDef.name(),
|
||||||
"state", "Running", assignedVm.get()));
|
"state", "Running", chosenVm));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Comparator<VmChannel> preferRunning
|
private static Comparator<VmChannel> preferRunning
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue