chylex/Minecraft-Phantom-Panel
1
0
Fork 0
mirror of https://github.com/chylex/Minecraft-Phantom-Panel.git synced 2025-05-05 03:34:05 +02:00
Code

Fix race conditions when transitioning instance states during Agent shutdown

Browse Source
This commit is contained in:
chylex 2023-01-25 02:23:58 +01:00
parent f4aec6f11d
commit 3c10e1a8f9
Signed by: chylex
GPG Key ID: 4DE42C8F19A80548
6 changed files with 58 additions and 49 deletions

27
Agent/Phantom.Agent.Services/Instances/Instance.cs
View File

@ -85,10 +85,10 @@ sealed class Instance : IDisposable {
}
}
public async Task<LaunchInstanceResult> Launch(CancellationToken cancellationToken) {
await stateTransitioningActionSemaphore.WaitAsync(cancellationToken);
public async Task<LaunchInstanceResult> Launch(CancellationToken shutdownCancellationToken) {
await stateTransitioningActionSemaphore.WaitAsync(shutdownCancellationToken);
try {
return TransitionStateAndReturn(currentState.Launch(new InstanceContextImpl(this)));
return TransitionStateAndReturn(currentState.Launch(new InstanceContextImpl(this, shutdownCancellationToken)));
} catch (Exception e) {
logger.Error(e, "Caught exception while launching instance.");
return LaunchInstanceResult.UnknownError;
@ -126,10 +126,13 @@ sealed class Instance : IDisposable {
private sealed class InstanceContextImpl : InstanceContext {
private readonly Instance instance;
private readonly CancellationToken shutdownCancellationToken;
private int statusUpdateCounter;
public InstanceContextImpl(Instance instance) : base(instance.Configuration, instance.Launcher) {
public InstanceContextImpl(Instance instance, CancellationToken shutdownCancellationToken) : base(instance.Configuration, instance.Launcher) {
this.instance = instance;
this.shutdownCancellationToken = shutdownCancellationToken;
}
public override LaunchServices LaunchServices => instance.launchServices;
@ -148,10 +151,20 @@ sealed class Instance : IDisposable {
});
}
public override void TransitionState(Func<IInstanceState> newState) {
instance.stateTransitioningActionSemaphore.Wait();
public override void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus) {
instance.stateTransitioningActionSemaphore.Wait(CancellationToken.None);
try {
instance.TransitionState(newState());
var (state, status) = newStateAndStatus();
if (state is not InstanceNotRunningState && shutdownCancellationToken.IsCancellationRequested) {
instance.logger.Verbose("Cancelled state transition to {State} due to Agent shutdown.", state.GetType().Name);
return;
}
if (status != null) {
ReportStatus(status);
}
instance.TransitionState(state);
} catch (Exception e) {
instance.logger.Error(e, "Caught exception during state transition.");
} finally {

6
Agent/Phantom.Agent.Services/Instances/InstanceContext.cs
View File

@ -20,9 +20,9 @@ abstract class InstanceContext {
}
public abstract void ReportStatus(IInstanceStatus newStatus);
public abstract void TransitionState(Func<IInstanceState> newState);
public abstract void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus);
public void TransitionState(IInstanceState newState) {
TransitionState(() => newState);
public void TransitionState(IInstanceState newState, IInstanceStatus? newStatus = null) {
TransitionState(() => (newState, newStatus));
}
}

44
Agent/Phantom.Agent.Services/Instances/InstanceSessionManager.cs
View File

@ -41,36 +41,36 @@ sealed class InstanceSessionManager : IDisposable {
this.shutdownCancellationToken = shutdownCancellationTokenSource.Token;
}
[SuppressMessage("ReSharper", "ConvertIfStatementToReturnStatement")]
private async Task<InstanceActionResult<T>> AcquireSemaphoreAndRunWithInstance<T>(Guid instanceGuid, Func<Instance, Task<T>> func) {
private async Task<InstanceActionResult<T>> AcquireSemaphoreAndRun<T>(Func<Task<InstanceActionResult<T>>> func) {
try {
await semaphore.WaitAsync(shutdownCancellationToken);
try {
return await func();
} finally {
semaphore.Release();
}
} catch (OperationCanceledException) {
return InstanceActionResult.General<T>(InstanceActionGeneralResult.AgentShuttingDown);
}
}
try {
if (!instances.TryGetValue(instanceGuid, out var instance)) {
return InstanceActionResult.General<T>(InstanceActionGeneralResult.InstanceDoesNotExist);
}
else {
[SuppressMessage("ReSharper", "ConvertIfStatementToReturnStatement")]
private Task<InstanceActionResult<T>> AcquireSemaphoreAndRunWithInstance<T>(Guid instanceGuid, Func<Instance, Task<T>> func) {
return AcquireSemaphoreAndRun(async () => {
if (instances.TryGetValue(instanceGuid, out var instance)) {
return InstanceActionResult.Concrete(await func(instance));
}
} finally {
semaphore.Release();
}
else {
return InstanceActionResult.General<T>(InstanceActionGeneralResult.InstanceDoesNotExist);
}
});
}
public async Task<InstanceActionResult<ConfigureInstanceResult>> Configure(InstanceConfiguration configuration) {
try {
await semaphore.WaitAsync(shutdownCancellationToken);
} catch (OperationCanceledException) {
return InstanceActionResult.General<ConfigureInstanceResult>(InstanceActionGeneralResult.AgentShuttingDown);
}
var instanceGuid = configuration.InstanceGuid;
try {
return await AcquireSemaphoreAndRun(async () => {
var instanceGuid = configuration.InstanceGuid;
var otherInstances = instances.Values.Where(inst => inst.Configuration.InstanceGuid != instanceGuid).ToArray();
if (otherInstances.Length + 1 > agentInfo.MaxInstances) {
return InstanceActionResult.Concrete(ConfigureInstanceResult.InstanceLimitExceeded);
@ -115,9 +115,7 @@ sealed class InstanceSessionManager : IDisposable {
}
return InstanceActionResult.Concrete(ConfigureInstanceResult.Success);
} finally {
semaphore.Release();
}
});
}
public Task<InstanceActionResult<LaunchInstanceResult>> Launch(Guid instanceGuid) {

5
Agent/Phantom.Agent.Services/Instances/States/InstanceLaunchingState.cs
View File

@ -66,11 +66,10 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
context.TransitionState(() => {
if (cancellationTokenSource.IsCancellationRequested) {
context.PortManager.Release(context.Configuration);
context.ReportStatus(InstanceStatus.NotRunning);
return new InstanceNotRunningState();
return (new InstanceNotRunningState(), InstanceStatus.NotRunning);
}
else {
return new InstanceRunningState(context, task.Result);
return (new InstanceRunningState(context, task.Result), null);
}
});
}

9
Agent/Phantom.Agent.Services/Instances/States/InstanceRunningState.cs
View File

@ -30,8 +30,7 @@ sealed class InstanceRunningState : IInstanceState {
if (session.HasEnded) {
if (sessionObjects.Dispose()) {
context.Logger.Warning("Session ended immediately after it was started.");
context.ReportStatus(InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError));
context.LaunchServices.TaskManager.Run(() => context.TransitionState(new InstanceNotRunningState()));
context.LaunchServices.TaskManager.Run(() => context.TransitionState(new InstanceNotRunningState(), InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)));
}
}
else {
@ -52,13 +51,11 @@ sealed class InstanceRunningState : IInstanceState {
if (isStopping) {
context.Logger.Information("Session ended.");
context.ReportStatus(InstanceStatus.NotRunning);
context.TransitionState(new InstanceNotRunningState());
context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
}
else {
context.Logger.Information("Session ended unexpectedly, restarting...");
context.ReportStatus(InstanceStatus.Restarting);
context.TransitionState(new InstanceLaunchingState(context));
context.TransitionState(new InstanceLaunchingState(context), InstanceStatus.Restarting);
}
}

16
Agent/Phantom.Agent.Services/Instances/States/InstanceStoppingState.cs
View File

@ -1,4 +1,5 @@
using Phantom.Agent.Minecraft.Command;
using System.Diagnostics;
using Phantom.Agent.Minecraft.Command;
using Phantom.Agent.Minecraft.Instance;
using Phantom.Common.Data.Instance;
using Phantom.Common.Data.Minecraft;
@ -32,26 +33,27 @@ sealed class InstanceStoppingState : IInstanceState, IDisposable {
await DoWaitForSessionToEnd();
} finally {
context.Logger.Information("Session stopped.");
context.ReportStatus(InstanceStatus.NotRunning);
context.TransitionState(new InstanceNotRunningState());
context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
}
}
private async Task DoSendStopCommand() {
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try {
await session.SendCommand(MinecraftCommand.Stop, cts.Token);
await session.SendCommand(MinecraftCommand.Stop, timeout.Token);
} catch (OperationCanceledException) {
// ignore
} catch (ObjectDisposedException e) when (e.ObjectName == typeof(Process).FullName && session.HasEnded) {
// ignore
} catch (Exception e) {
context.Logger.Warning(e, "Caught exception while sending stop command.");
}
}
private async Task DoWaitForSessionToEnd() {
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(55));
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(55));
try {
await session.WaitForExit(cts.Token);
await session.WaitForExit(timeout.Token);
} catch (OperationCanceledException) {
try {
context.Logger.Warning("Waiting timed out, killing session...");