WIP

Agent/Phantom.Agent.Minecraft/Server/MinecraftServerExecutableDownloader.cs

@@ -16,36 +16,47 @@ sealed class MinecraftServerExecutableDownloader {
 	public event EventHandler? Completed;
 	
 	private readonly CancellationTokenSource cancellationTokenSource = new ();
-	private int listeners = 0;
+	
+	private readonly List<CancellationTokenRegistration> listenerCancellationRegistrations = new ();
+	private int listenerCount = 0;
 
 	public MinecraftServerExecutableDownloader(FileDownloadInfo fileDownloadInfo, string minecraftVersion, string filePath, MinecraftServerExecutableDownloadListener listener) {
 		Register(listener);
-		Task = DownloadAndGetPath(fileDownloadInfo, minecraftVersion, filePath);
+		Task = DownloadAndGetPath(fileDownloadInfo, minecraftVersion, filePath, new DownloadProgressCallback(this), cancellationTokenSource.Token);
 		Task.ContinueWith(OnCompleted, TaskScheduler.Default);
 	}
 
 	public void Register(MinecraftServerExecutableDownloadListener listener) {
-		++listeners;
-		Logger.Debug("Registered download listener, current listener count: {Listeners}", listeners);
+		int newListenerCount;
 		
-		DownloadProgress += listener.DownloadProgressEventHandler;
-		listener.CancellationToken.Register(Unregister, listener);
+		lock (this) {
+			newListenerCount = ++listenerCount;
+			
+			DownloadProgress += listener.DownloadProgressEventHandler;
+			listenerCancellationRegistrations.Add(listener.CancellationToken.Register(Unregister, listener));
+		}
+		
+		Logger.Debug("Registered download listener, current listener count: {Listeners}", newListenerCount);
 	}
 
 	private void Unregister(object? listenerObject) {
-		MinecraftServerExecutableDownloadListener listener = (MinecraftServerExecutableDownloadListener) listenerObject!;
-		DownloadProgress -= listener.DownloadProgressEventHandler;
+		int newListenerCount;
+		
+		lock (this) {
+			MinecraftServerExecutableDownloadListener listener = (MinecraftServerExecutableDownloadListener) listenerObject!;
+			DownloadProgress -= listener.DownloadProgressEventHandler;
 
-		if (--listeners <= 0) {
-			Logger.Debug("Unregistered last download listener, cancelling download.");
-			try {
+			newListenerCount = --listenerCount;
+			if (newListenerCount <= 0) {
 				cancellationTokenSource.Cancel();
-			} catch (ObjectDisposedException) {
-				// TODO THIS IS BAD
 			}
 		}
+		
+		if (newListenerCount <= 0) {
+			Logger.Debug("Unregistered last download listener, cancelling download.");
+		}
 		else {
-			Logger.Debug("Unregistered download listener, current listener count: {Listeners}", listeners);
+			Logger.Debug("Unregistered download listener, current listener count: {Listeners}", newListenerCount);
 		}
 	}
 
@@ -55,9 +66,19 @@ sealed class MinecraftServerExecutableDownloader {
 
 	private void OnCompleted(Task task) {
 		Logger.Debug("Download task completed.");
-		Completed?.Invoke(this, EventArgs.Empty);
-		Completed = null;
-		DownloadProgress = null;
+
+		lock (this) {
+			Completed?.Invoke(this, EventArgs.Empty);
+			Completed = null;
+			DownloadProgress = null;
+
+			foreach (var registration in listenerCancellationRegistrations) {
+				registration.Dispose();
+			}
+			
+			listenerCancellationRegistrations.Clear();
+			cancellationTokenSource.Dispose();
+		}
 	}
 
 	private sealed class DownloadProgressCallback {
@@ -72,15 +93,14 @@ sealed class MinecraftServerExecutableDownloader {
 		}
 	}
 
-	private async Task<string?> DownloadAndGetPath(FileDownloadInfo fileDownloadInfo, string minecraftVersion, string filePath) {
+	private static async Task<string?> DownloadAndGetPath(FileDownloadInfo fileDownloadInfo, string minecraftVersion, string filePath, DownloadProgressCallback progressCallback, CancellationToken cancellationToken) {
 		string tmpFilePath = filePath + ".tmp";
 
-		var cancellationToken = cancellationTokenSource.Token;
 		try {
 			Logger.Information("Downloading server version {Version} from: {Url} ({Size})", minecraftVersion, fileDownloadInfo.DownloadUrl, fileDownloadInfo.Size.ToHumanReadable(decimalPlaces: 1));
 			try {
 				using var http = new HttpClient();
-				await FetchServerExecutableFile(http, new DownloadProgressCallback(this), fileDownloadInfo, tmpFilePath, cancellationToken);
+				await FetchServerExecutableFile(http, progressCallback, fileDownloadInfo, tmpFilePath, cancellationToken);
 			} catch (Exception) {
 				TryDeleteExecutableAfterFailure(tmpFilePath);
 				throw;
@@ -98,8 +118,6 @@ sealed class MinecraftServerExecutableDownloader {
 		} catch (Exception e) {
 			Logger.Error(e, "An unexpected error occurred.");
 			return null;
-		} finally {
-			cancellationTokenSource.Dispose();
 		}
 	}
 

Agent/Phantom.Agent.Services/Backups/BackupManager.cs

@@ -81,6 +81,7 @@ sealed class BackupManager : IDisposable {
 				} catch (OperationCanceledException) {
 					// Ignore.
 				} catch (TimeoutException) {
+					resultBuilder.Warnings |= BackupCreationWarnings.CouldNotRestoreAutomaticSaving;
 					logger.Warning("Timed out waiting for automatic saving to be re-enabled.");
 				} catch (Exception e) {
 					resultBuilder.Warnings |= BackupCreationWarnings.CouldNotRestoreAutomaticSaving;

Agent/Phantom.Agent.Services/Backups/BackupServerCommandDispatcher.cs

@@ -1,4 +1,5 @@
-using System.Text.RegularExpressions;
+using System.Collections.Immutable;
+using System.Text.RegularExpressions;
 using Phantom.Agent.Minecraft.Command;
 using Phantom.Agent.Minecraft.Instance;
 using Phantom.Utils.Tasks;
@@ -7,8 +8,26 @@ using Serilog;
 namespace Phantom.Agent.Services.Backups;
 
 sealed partial class BackupServerCommandDispatcher : IDisposable {
-	[GeneratedRegex(@"^\[(?:.*?)\] \[Server thread/INFO\](?:.*?): (.*?)$", RegexOptions.NonBacktracking)]
+	[GeneratedRegex(@"^(?:(?:\[.*?\] \[Server thread/INFO\].*?:)|(?:[\d-]+? [\d:]+? \[INFO\])) (.*?)$", RegexOptions.NonBacktracking)]
 	private static partial Regex ServerThreadInfoRegex();
+	
+	private static readonly ImmutableHashSet<string> AutomaticSavingDisabledMessages = ImmutableHashSet.Create(
+		"Automatic saving is now disabled",
+		"Turned off world auto-saving",
+		"CONSOLE: Disabling level saving.."
+	);
+	
+	private static readonly ImmutableHashSet<string> SavedTheGameMessages = ImmutableHashSet.Create(
+		"Saved the game",
+		"Saved the world",
+		"CONSOLE: Save complete."
+	);
+	
+	private static readonly ImmutableHashSet<string> AutomaticSavingEnabledMessages = ImmutableHashSet.Create(
+		"Automatic saving is now enabled",
+		"Turned on world auto-saving",
+		"CONSOLE: Enabling level saving.."
+	);
 
 	private readonly ILogger logger;
 	private readonly InstanceProcess process;
@@ -59,19 +78,19 @@ sealed partial class BackupServerCommandDispatcher : IDisposable {
 		string info = match.Groups[1].Value;
 
 		if (!automaticSavingDisabled.Task.IsCompleted) {
-			if (info == "Automatic saving is now disabled") {
+			if (AutomaticSavingDisabledMessages.Contains(info)) {
 				logger.Debug("Detected that automatic saving is disabled.");
 				automaticSavingDisabled.SetResult();
 			}
 		}
 		else if (!savedTheGame.Task.IsCompleted) {
-			if (info == "Saved the game") {
+			if (SavedTheGameMessages.Contains(info)) {
 				logger.Debug("Detected that the game is saved.");
 				savedTheGame.SetResult();
 			}
 		}
 		else if (!automaticSavingEnabled.Task.IsCompleted) {
-			if (info == "Automatic saving is now enabled") {
+			if (AutomaticSavingEnabledMessages.Contains(info)) {
 				logger.Debug("Detected that automatic saving is enabled.");
 				automaticSavingEnabled.SetResult();
 			}

Agent/Phantom.Agent.Services/Instances/InstanceActor.cs

@@ -135,7 +135,12 @@ sealed class InstanceActor : ReceiveActor<InstanceActor.ICommand> {
 			return new BackupCreationResult(BackupCreationResultKind.InstanceNotRunning);
 		}
 		else {
-			return await command.BackupManager.CreateBackup(context.ShortName, runningState.Process, shutdownCancellationToken);
+			SetAndReportStatus(InstanceStatus.BackingUp);
+			try {
+				return await command.BackupManager.CreateBackup(context.ShortName, runningState.Process, shutdownCancellationToken);
+			} finally {
+				SetAndReportStatus(InstanceStatus.Running);
+			}
 		}
 	}
 

Common/Phantom.Common.Data/Instance/IInstanceStatus.cs

@@ -9,9 +9,10 @@ namespace Phantom.Common.Data.Instance;
 [MemoryPackUnion(3, typeof(InstanceIsDownloading))]
 [MemoryPackUnion(4, typeof(InstanceIsLaunching))]
 [MemoryPackUnion(5, typeof(InstanceIsRunning))]
-[MemoryPackUnion(6, typeof(InstanceIsRestarting))]
-[MemoryPackUnion(7, typeof(InstanceIsStopping))]
-[MemoryPackUnion(8, typeof(InstanceIsFailed))]
+[MemoryPackUnion(6, typeof(InstanceIsBackingUp))]
+[MemoryPackUnion(7, typeof(InstanceIsRestarting))]
+[MemoryPackUnion(8, typeof(InstanceIsStopping))]
+[MemoryPackUnion(9, typeof(InstanceIsFailed))]
 public partial interface IInstanceStatus {}
 
 [MemoryPackable(GenerateType.VersionTolerant)]
@@ -32,6 +33,9 @@ public sealed partial record InstanceIsLaunching : IInstanceStatus;
 [MemoryPackable(GenerateType.VersionTolerant)]
 public sealed partial record InstanceIsRunning : IInstanceStatus;
 
+[MemoryPackable(GenerateType.VersionTolerant)]
+public sealed partial record InstanceIsBackingUp : IInstanceStatus;
+
 [MemoryPackable(GenerateType.VersionTolerant)]
 public sealed partial record InstanceIsRestarting : IInstanceStatus;
 
@@ -46,6 +50,7 @@ public static class InstanceStatus {
 	public static readonly IInstanceStatus NotRunning = new InstanceIsNotRunning();
 	public static readonly IInstanceStatus Launching = new InstanceIsLaunching();
 	public static readonly IInstanceStatus Running = new InstanceIsRunning();
+	public static readonly IInstanceStatus BackingUp = new InstanceIsBackingUp();
 	public static readonly IInstanceStatus Restarting = new InstanceIsRestarting();
 	public static readonly IInstanceStatus Stopping = new InstanceIsStopping();
 	
@@ -58,7 +63,7 @@ public static class InstanceStatus {
 	}
 
 	public static bool IsRunning(this IInstanceStatus status) {
-		return status is InstanceIsRunning;
+		return status is InstanceIsRunning or InstanceIsBackingUp;
 	}
 	
 	public static bool IsStopping(this IInstanceStatus status) {
@@ -70,10 +75,10 @@ public static class InstanceStatus {
 	}
 
 	public static bool CanStop(this IInstanceStatus status) {
-		return status is InstanceIsDownloading or InstanceIsLaunching or InstanceIsRunning;
+		return status.IsRunning() || status.IsLaunching();
 	}
 
 	public static bool CanSendCommand(this IInstanceStatus status) {
-		return status is InstanceIsRunning;
+		return status.IsRunning();
 	}
 }

Controller/Phantom.Controller.Services/Agents/AgentActor.cs

@@ -13,11 +13,13 @@ using Phantom.Common.Data.Web.Minecraft;
 using Phantom.Common.Messages.Agent;
 using Phantom.Common.Messages.Agent.ToAgent;
 using Phantom.Controller.Database;
+using Phantom.Controller.Database.Entities;
 using Phantom.Controller.Minecraft;
 using Phantom.Controller.Services.Instances;
 using Phantom.Utils.Actor;
 using Phantom.Utils.Actor.Mailbox;
 using Phantom.Utils.Actor.Tasks;
+using Phantom.Utils.Collections;
 using Phantom.Utils.Logging;
 using Phantom.Utils.Rpc.Runtime;
 using Serilog;
@@ -194,21 +196,29 @@ sealed class AgentActor : ReceiveActor<AgentActor.ICommand> {
 	public sealed record ReceiveInstanceDataCommand(Instance Instance) : ICommand, IJumpAhead;
 
 	private async Task Initialize(InitializeCommand command) {
-		await using var ctx = dbProvider.Eager();
-		await foreach (var entity in ctx.Instances.Where(instance => instance.AgentGuid == agentGuid).AsAsyncEnumerable().WithCancellation(cancellationToken)) {
+		ImmutableArray<InstanceEntity> instanceEntities;
+		await using (var ctx = dbProvider.Eager()) {
+			instanceEntities = await ctx.Instances.Where(instance => instance.AgentGuid == agentGuid).AsAsyncEnumerable().ToImmutableArrayCatchingExceptionsAsync(OnException, cancellationToken);
+		}
+
+		static void OnException(Exception e) {
+			Logger.Error(e, "Could not load instance from database.");
+		}
+
+		foreach (var instanceEntity in instanceEntities) {
 			var instanceConfiguration = new InstanceConfiguration(
-				entity.AgentGuid,
-				entity.InstanceName,
-				entity.ServerPort,
-				entity.RconPort,
-				entity.MinecraftVersion,
-				entity.MinecraftServerKind,
-				entity.MemoryAllocation,
-				entity.JavaRuntimeGuid,
-				JvmArgumentsHelper.Split(entity.JvmArguments)
+				instanceEntity.AgentGuid,
+				instanceEntity.InstanceName,
+				instanceEntity.ServerPort,
+				instanceEntity.RconPort,
+				instanceEntity.MinecraftVersion,
+				instanceEntity.MinecraftServerKind,
+				instanceEntity.MemoryAllocation,
+				instanceEntity.JavaRuntimeGuid,
+				JvmArgumentsHelper.Split(instanceEntity.JvmArguments)
 			);
 
-			CreateNewInstance(Instance.Offline(entity.InstanceGuid, instanceConfiguration, entity.LaunchAutomatically));
+			CreateNewInstance(Instance.Offline(instanceEntity.InstanceGuid, instanceConfiguration, instanceEntity.LaunchAutomatically));
 		}
 	}
 

Utils/Phantom.Utils/Collections/EnumerableExtensions.cs

@@ -12,6 +12,27 @@ public static class EnumerableExtensions {
 
 		return builder.ToImmutable();
 	}
+
+	public static async Task<ImmutableArray<TSource>> ToImmutableArrayCatchingExceptionsAsync<TSource>(this IAsyncEnumerable<TSource> source, Action<Exception> onException, CancellationToken cancellationToken = default) {
+		var builder = ImmutableArray.CreateBuilder<TSource>();
+		
+		await using (var enumerator = source.GetAsyncEnumerator(cancellationToken)) {
+			while (true) {
+				try {
+					if (!await enumerator.MoveNextAsync()) {
+						break;
+					}
+				} catch (Exception e) {
+					onException(e);
+					continue;
+				}
+
+				builder.Add(enumerator.Current);
+			}
+		}
+
+		return builder.ToImmutable();
+	}
 	
 	public static async Task<ImmutableHashSet<TSource>> ToImmutableSetAsync<TSource>(this IAsyncEnumerable<TSource> source, CancellationToken cancellationToken = default) {
 		var builder = ImmutableHashSet.CreateBuilder<TSource>();

Web/Phantom.Web/Shared/InstanceCommandInput.razor

@@ -51,6 +51,7 @@
       form.SubmitModel.StopSubmitting(result.Map(Messages.ToSentence, InstanceActionFailureExtensions.ToSentence));
     }
 
+    StateHasChanged();
     await commandInputElement.FocusAsync(preventScroll: true);
   }
 

Web/Phantom.Web/Shared/InstanceStatusText.razor

@@ -28,6 +28,11 @@
     <span class="fw-semibold text-success">Running</span>
     break;
 
+  case InstanceIsBackingUp:
+    <div class="spinner-border" role="status"></div>
+    <span class="fw-semibold">&nbsp;Backing Up</span>
+    break;
+  
   case InstanceIsRestarting:
     <div class="spinner-border" role="status"></div>
     <span class="fw-semibold">&nbsp;Restarting</span>
@@ -41,6 +46,10 @@
   case InstanceIsFailed failed:
     <span class="fw-semibold text-danger">Failed <sup title="@failed.Reason.ToSentence()">[?]</sup></span>
     break;
+    
+  default:
+    <span class="fw-semibold">Unknown</span>
+    break;
 }
 </nobr>