chylex/Minecraft-Phantom-Panel
1
0
Fork 0
mirror of https://github.com/chylex/Minecraft-Phantom-Panel.git synced 2025-05-30 04:34:04 +02:00
Code

Fix race conditions when transitioning instance states during Agent shutdown

Browse Source
This commit is contained in:
chylex 2023-01-25 02:23:58 +01:00
parent f4aec6f11d
commit 3c10e1a8f9
Signed by: chylex
GPG Key ID: 4DE42C8F19A80548
6 changed files with 58 additions and 49 deletions

27
Agent/Phantom.Agent.Services/Instances/Instance.cs
View File

@ -85,10 +85,10 @@ sealed class Instance : IDisposable {
} }
} }
public async Task<LaunchInstanceResult> Launch(CancellationToken cancellationToken) { public async Task<LaunchInstanceResult> Launch(CancellationToken shutdownCancellationToken) {
await stateTransitioningActionSemaphore.WaitAsync(cancellationToken); await stateTransitioningActionSemaphore.WaitAsync(shutdownCancellationToken);
try { try {
return TransitionStateAndReturn(currentState.Launch(new InstanceContextImpl(this))); return TransitionStateAndReturn(currentState.Launch(new InstanceContextImpl(this, shutdownCancellationToken)));
} catch (Exception e) { } catch (Exception e) {
logger.Error(e, "Caught exception while launching instance."); logger.Error(e, "Caught exception while launching instance.");
return LaunchInstanceResult.UnknownError; return LaunchInstanceResult.UnknownError;
@ -126,10 +126,13 @@ sealed class Instance : IDisposable {
private sealed class InstanceContextImpl : InstanceContext { private sealed class InstanceContextImpl : InstanceContext {
private readonly Instance instance; private readonly Instance instance;
private readonly CancellationToken shutdownCancellationToken;
private int statusUpdateCounter; private int statusUpdateCounter;
public InstanceContextImpl(Instance instance) : base(instance.Configuration, instance.Launcher) { public InstanceContextImpl(Instance instance, CancellationToken shutdownCancellationToken) : base(instance.Configuration, instance.Launcher) {
this.instance = instance; this.instance = instance;
this.shutdownCancellationToken = shutdownCancellationToken;
} }
public override LaunchServices LaunchServices => instance.launchServices; public override LaunchServices LaunchServices => instance.launchServices;
@ -148,10 +151,20 @@ sealed class Instance : IDisposable {
}); });
} }
public override void TransitionState(Func<IInstanceState> newState) { public override void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus) {
instance.stateTransitioningActionSemaphore.Wait(); instance.stateTransitioningActionSemaphore.Wait(CancellationToken.None);
try { try {
instance.TransitionState(newState()); var (state, status) = newStateAndStatus();
if (state is not InstanceNotRunningState && shutdownCancellationToken.IsCancellationRequested) {
instance.logger.Verbose("Cancelled state transition to {State} due to Agent shutdown.", state.GetType().Name);
return;
}
if (status != null) {
ReportStatus(status);
}
instance.TransitionState(state);
} catch (Exception e) { } catch (Exception e) {
instance.logger.Error(e, "Caught exception during state transition."); instance.logger.Error(e, "Caught exception during state transition.");
} finally { } finally {

6
Agent/Phantom.Agent.Services/Instances/InstanceContext.cs
View File

@ -20,9 +20,9 @@ abstract class InstanceContext {
} }
public abstract void ReportStatus(IInstanceStatus newStatus); public abstract void ReportStatus(IInstanceStatus newStatus);
public abstract void TransitionState(Func<IInstanceState> newState); public abstract void TransitionState(Func<(IInstanceState, IInstanceStatus?)> newStateAndStatus);
public void TransitionState(IInstanceState newState) { public void TransitionState(IInstanceState newState, IInstanceStatus? newStatus = null) {
TransitionState(() => newState); TransitionState(() => (newState, newStatus));
} }
} }

44
Agent/Phantom.Agent.Services/Instances/InstanceSessionManager.cs
View File

@ -41,36 +41,36 @@ sealed class InstanceSessionManager : IDisposable {
this.shutdownCancellationToken = shutdownCancellationTokenSource.Token; this.shutdownCancellationToken = shutdownCancellationTokenSource.Token;
} }
[SuppressMessage("ReSharper", "ConvertIfStatementToReturnStatement")] private async Task<InstanceActionResult<T>> AcquireSemaphoreAndRun<T>(Func<Task<InstanceActionResult<T>>> func) {
private async Task<InstanceActionResult<T>> AcquireSemaphoreAndRunWithInstance<T>(Guid instanceGuid, Func<Instance, Task<T>> func) {
try { try {
await semaphore.WaitAsync(shutdownCancellationToken); await semaphore.WaitAsync(shutdownCancellationToken);
try {
return await func();
} finally {
semaphore.Release();
}
} catch (OperationCanceledException) { } catch (OperationCanceledException) {
return InstanceActionResult.General<T>(InstanceActionGeneralResult.AgentShuttingDown); return InstanceActionResult.General<T>(InstanceActionGeneralResult.AgentShuttingDown);
} }
}
try { [SuppressMessage("ReSharper", "ConvertIfStatementToReturnStatement")]
if (!instances.TryGetValue(instanceGuid, out var instance)) { private Task<InstanceActionResult<T>> AcquireSemaphoreAndRunWithInstance<T>(Guid instanceGuid, Func<Instance, Task<T>> func) {
return InstanceActionResult.General<T>(InstanceActionGeneralResult.InstanceDoesNotExist); return AcquireSemaphoreAndRun(async () => {
} if (instances.TryGetValue(instanceGuid, out var instance)) {
else {
return InstanceActionResult.Concrete(await func(instance)); return InstanceActionResult.Concrete(await func(instance));
} }
} finally { else {
semaphore.Release(); return InstanceActionResult.General<T>(InstanceActionGeneralResult.InstanceDoesNotExist);
} }
});
} }
public async Task<InstanceActionResult<ConfigureInstanceResult>> Configure(InstanceConfiguration configuration) { public async Task<InstanceActionResult<ConfigureInstanceResult>> Configure(InstanceConfiguration configuration) {
try { return await AcquireSemaphoreAndRun(async () => {
await semaphore.WaitAsync(shutdownCancellationToken); var instanceGuid = configuration.InstanceGuid;
} catch (OperationCanceledException) {
return InstanceActionResult.General<ConfigureInstanceResult>(InstanceActionGeneralResult.AgentShuttingDown);
}
var instanceGuid = configuration.InstanceGuid;
try {
var otherInstances = instances.Values.Where(inst => inst.Configuration.InstanceGuid != instanceGuid).ToArray(); var otherInstances = instances.Values.Where(inst => inst.Configuration.InstanceGuid != instanceGuid).ToArray();
if (otherInstances.Length + 1 > agentInfo.MaxInstances) { if (otherInstances.Length + 1 > agentInfo.MaxInstances) {
return InstanceActionResult.Concrete(ConfigureInstanceResult.InstanceLimitExceeded); return InstanceActionResult.Concrete(ConfigureInstanceResult.InstanceLimitExceeded);
@ -115,9 +115,7 @@ sealed class InstanceSessionManager : IDisposable {
} }
return InstanceActionResult.Concrete(ConfigureInstanceResult.Success); return InstanceActionResult.Concrete(ConfigureInstanceResult.Success);
} finally { });
semaphore.Release();
}
} }
public Task<InstanceActionResult<LaunchInstanceResult>> Launch(Guid instanceGuid) { public Task<InstanceActionResult<LaunchInstanceResult>> Launch(Guid instanceGuid) {

5
Agent/Phantom.Agent.Services/Instances/States/InstanceLaunchingState.cs
View File

@ -66,11 +66,10 @@ sealed class InstanceLaunchingState : IInstanceState, IDisposable {
context.TransitionState(() => { context.TransitionState(() => {
if (cancellationTokenSource.IsCancellationRequested) { if (cancellationTokenSource.IsCancellationRequested) {
context.PortManager.Release(context.Configuration); context.PortManager.Release(context.Configuration);
context.ReportStatus(InstanceStatus.NotRunning); return (new InstanceNotRunningState(), InstanceStatus.NotRunning);
return new InstanceNotRunningState();
} }
else { else {
return new InstanceRunningState(context, task.Result); return (new InstanceRunningState(context, task.Result), null);
} }
}); });
} }

9
Agent/Phantom.Agent.Services/Instances/States/InstanceRunningState.cs
View File

@ -30,8 +30,7 @@ sealed class InstanceRunningState : IInstanceState {
if (session.HasEnded) { if (session.HasEnded) {
if (sessionObjects.Dispose()) { if (sessionObjects.Dispose()) {
context.Logger.Warning("Session ended immediately after it was started."); context.Logger.Warning("Session ended immediately after it was started.");
context.ReportStatus(InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)); context.LaunchServices.TaskManager.Run(() => context.TransitionState(new InstanceNotRunningState(), InstanceStatus.Failed(InstanceLaunchFailReason.UnknownError)));
context.LaunchServices.TaskManager.Run(() => context.TransitionState(new InstanceNotRunningState()));
} }
} }
else { else {
@ -52,13 +51,11 @@ sealed class InstanceRunningState : IInstanceState {
if (isStopping) { if (isStopping) {
context.Logger.Information("Session ended."); context.Logger.Information("Session ended.");
context.ReportStatus(InstanceStatus.NotRunning); context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
context.TransitionState(new InstanceNotRunningState());
} }
else { else {
context.Logger.Information("Session ended unexpectedly, restarting..."); context.Logger.Information("Session ended unexpectedly, restarting...");
context.ReportStatus(InstanceStatus.Restarting); context.TransitionState(new InstanceLaunchingState(context), InstanceStatus.Restarting);
context.TransitionState(new InstanceLaunchingState(context));
} }
} }

16
Agent/Phantom.Agent.Services/Instances/States/InstanceStoppingState.cs
View File

@ -1,4 +1,5 @@
using Phantom.Agent.Minecraft.Command; using System.Diagnostics;
using Phantom.Agent.Minecraft.Command;
using Phantom.Agent.Minecraft.Instance; using Phantom.Agent.Minecraft.Instance;
using Phantom.Common.Data.Instance; using Phantom.Common.Data.Instance;
using Phantom.Common.Data.Minecraft; using Phantom.Common.Data.Minecraft;
@ -32,26 +33,27 @@ sealed class InstanceStoppingState : IInstanceState, IDisposable {
await DoWaitForSessionToEnd(); await DoWaitForSessionToEnd();
} finally { } finally {
context.Logger.Information("Session stopped."); context.Logger.Information("Session stopped.");
context.ReportStatus(InstanceStatus.NotRunning); context.TransitionState(new InstanceNotRunningState(), InstanceStatus.NotRunning);
context.TransitionState(new InstanceNotRunningState());
} }
} }
private async Task DoSendStopCommand() { private async Task DoSendStopCommand() {
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
try { try {
await session.SendCommand(MinecraftCommand.Stop, cts.Token); await session.SendCommand(MinecraftCommand.Stop, timeout.Token);
} catch (OperationCanceledException) { } catch (OperationCanceledException) {
// ignore // ignore
} catch (ObjectDisposedException e) when (e.ObjectName == typeof(Process).FullName && session.HasEnded) {
// ignore
} catch (Exception e) { } catch (Exception e) {
context.Logger.Warning(e, "Caught exception while sending stop command."); context.Logger.Warning(e, "Caught exception while sending stop command.");
} }
} }
private async Task DoWaitForSessionToEnd() { private async Task DoWaitForSessionToEnd() {
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(55)); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(55));
try { try {
await session.WaitForExit(cts.Token); await session.WaitForExit(timeout.Token);
} catch (OperationCanceledException) { } catch (OperationCanceledException) {
try { try {
context.Logger.Warning("Waiting timed out, killing session..."); context.Logger.Warning("Waiting timed out, killing session...");