Add backend code for creating world backups (tar + zstd)

Agent/Phantom.Agent.Minecraft/Command/MinecraftCommand.cs

@@ -1,9 +1,15 @@
 namespace Phantom.Agent.Minecraft.Command; 
 
 public static class MinecraftCommand {
+	public const string SaveOn = "save-on";
+	public const string SaveOff = "save-off";
 	public const string Stop = "stop";
-
+	
 	public static string Say(string message) {
 		return "say " + message;
 	}
+
+	public static string SaveAll(bool flush) {
+		return flush ? "save-all flush" : "save-all";
+	}
 }

Agent/Phantom.Agent.Minecraft/Instance/InstanceProperties.cs

@@ -5,6 +5,7 @@ using Phantom.Agent.Minecraft.Properties;
 namespace Phantom.Agent.Minecraft.Instance;
 
 public sealed record InstanceProperties(
+	Guid InstanceGuid,
 	Guid JavaRuntimeGuid,
 	JvmProperties JvmProperties,
 	ImmutableArray<string> JvmArguments,

Agent/Phantom.Agent.Minecraft/Instance/InstanceSession.cs

@@ -4,6 +4,8 @@ using Phantom.Utils.Collections;
 namespace Phantom.Agent.Minecraft.Instance; 
 
 public sealed class InstanceSession : IDisposable {
+	public InstanceProperties InstanceProperties { get; }
+	
 	private readonly RingBuffer<string> outputBuffer = new (10000);
 	private event EventHandler<string>? OutputEvent;
 
@@ -12,7 +14,8 @@ public sealed class InstanceSession : IDisposable {
 
 	private readonly Process process;
 
-	internal InstanceSession(Process process) {
+	internal InstanceSession(InstanceProperties instanceProperties, Process process) {
+		this.InstanceProperties = instanceProperties;
 		this.process = process;
 		this.process.EnableRaisingEvents = true;
 		this.process.Exited += ProcessOnExited;

Agent/Phantom.Agent.Minecraft/Launcher/BaseLauncher.cs

@@ -51,7 +51,7 @@ public abstract class BaseLauncher {
 		processArguments.Add("nogui");
 
 		var process = new Process { StartInfo = startInfo };
-		var session = new InstanceSession(process);
+		var session = new InstanceSession(instanceProperties, process);
 
 		try {
 			await AcceptEula(instanceProperties);

Agent/Phantom.Agent.Minecraft/Phantom.Agent.Minecraft.csproj

@@ -17,7 +17,6 @@
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Collections\Phantom.Utils.Collections.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.Cryptography\Phantom.Utils.Cryptography.csproj" />
     <ProjectReference Include="..\..\Utils\Phantom.Utils.IO\Phantom.Utils.IO.csproj" />
-    <ProjectReference Include="..\..\Utils\Phantom.Utils.Runtime\Phantom.Utils.Runtime.csproj" />
   </ItemGroup>
 
 </Project>

Agent/Phantom.Agent.Services/AgentFolders.cs

@@ -9,6 +9,7 @@ public sealed class AgentFolders {
 
 	public string DataFolderPath { get; }
 	public string InstancesFolderPath { get; }
+	public string BackupsFolderPath { get; }
 
 	public string TemporaryFolderPath { get; }
 	public string ServerExecutableFolderPath { get; }
@@ -18,6 +19,7 @@ public sealed class AgentFolders {
 	public AgentFolders(string dataFolderPath, string temporaryFolderPath, string javaSearchFolderPath) {
 		this.DataFolderPath = Path.GetFullPath(dataFolderPath);
 		this.InstancesFolderPath = Path.Combine(DataFolderPath, "instances");
+		this.BackupsFolderPath = Path.Combine(DataFolderPath, "backups");
 		
 		this.TemporaryFolderPath = Path.GetFullPath(temporaryFolderPath);
 		this.ServerExecutableFolderPath = Path.Combine(TemporaryFolderPath, "servers");
@@ -28,6 +30,7 @@ public sealed class AgentFolders {
 	public bool TryCreate() {
 		return TryCreateFolder(DataFolderPath) &&
 		       TryCreateFolder(InstancesFolderPath) &&
+		       TryCreateFolder(BackupsFolderPath) &&
 		       TryCreateFolder(TemporaryFolderPath) &&
 		       TryCreateFolder(ServerExecutableFolderPath);
 	}

Agent/Phantom.Agent.Services/AgentServices.cs

@@ -1,4 +1,5 @@
 using Phantom.Agent.Minecraft.Java;
+using Phantom.Agent.Services.Backups;
 using Phantom.Agent.Services.Instances;
 using Phantom.Common.Data.Agent;
 using Phantom.Common.Logging;
@@ -12,6 +13,7 @@ public sealed class AgentServices {
 	
 	private AgentFolders AgentFolders { get; }
 	private TaskManager TaskManager { get; }
+	private BackupManager BackupManager { get; }
 
 	internal JavaRuntimeRepository JavaRuntimeRepository { get; }
 	internal InstanceSessionManager InstanceSessionManager { get; }
@@ -19,6 +21,7 @@ public sealed class AgentServices {
 	public AgentServices(AgentInfo agentInfo, AgentFolders agentFolders) {
 		this.AgentFolders = agentFolders;
 		this.TaskManager = new TaskManager(PhantomLogger.Create<TaskManager, AgentServices>());
+		this.BackupManager = new BackupManager(agentFolders);
 		this.JavaRuntimeRepository = new JavaRuntimeRepository();
 		this.InstanceSessionManager = new InstanceSessionManager(agentInfo, agentFolders, JavaRuntimeRepository, TaskManager);
 	}

Agent/Phantom.Agent.Services/Backups/BackupArchiver.cs

@@ -0,0 +1,166 @@
+using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
+using System.Formats.Tar;
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Common.Data.Backups;
+using Phantom.Common.Logging;
+using Phantom.Utils.IO;
+using Serilog;
+
+namespace Phantom.Agent.Services.Backups; 
+
+sealed class BackupArchiver {
+	private readonly ILogger logger;
+	private readonly InstanceProperties instanceProperties;
+	private readonly CancellationToken cancellationToken;
+	
+	public BackupArchiver(string loggerName, InstanceProperties instanceProperties, CancellationToken cancellationToken) {
+		this.logger = PhantomLogger.Create<BackupArchiver>(loggerName);
+		this.instanceProperties = instanceProperties;
+		this.cancellationToken = cancellationToken;
+	}
+
+	private bool IsFolderSkipped(ImmutableList<string> relativePath) {
+		return relativePath is ["cache" or "crash-reports" or "debug" or "libraries" or "logs" or "mods" or "versions"];
+	}
+	
+	[SuppressMessage("ReSharper", "ConvertIfStatementToReturnStatement")]
+	private bool IsFileSkipped(ImmutableList<string> relativePath) {
+		var name = relativePath[^1];
+		
+		if (relativePath.Count == 2 && name == "session.lock") {
+			return true;
+		}
+
+		var extension = Path.GetExtension(name);
+		if (extension is ".jar" or ".zip") {
+			return true;
+		}
+		
+		return false;
+	}
+
+	public async Task ArchiveWorld(string destinationPath, BackupCreationResult.Builder resultBuilder) {
+		string backupFolderPath = Path.Combine(destinationPath, instanceProperties.InstanceGuid.ToString());
+		string backupFilePath = Path.Combine(backupFolderPath, DateTime.Now.ToString("yyyyMMdd-HHmmss") + ".tar");
+		
+		if (File.Exists(backupFilePath)) {
+			resultBuilder.Kind = BackupCreationResultKind.BackupAlreadyExists;
+			logger.Warning("Skipping backup, file already exists: {File}", backupFilePath);
+			return;
+		}
+		
+		try {
+			Directories.Create(backupFolderPath, Chmod.URWX_GRX);
+		} catch (Exception e) {
+			resultBuilder.Kind = BackupCreationResultKind.CouldNotCreateBackupFolder;
+			logger.Error(e, "Could not create backup folder: {Folder}", backupFolderPath);
+			return;
+		}
+
+		if (!await CopyWorldAndCreateTarArchive(backupFolderPath, backupFilePath, resultBuilder)) {
+			return;
+		}
+		
+		var compressedFilePath = await BackupCompressor.Compress(backupFilePath, cancellationToken);
+		if (compressedFilePath == null) {
+			resultBuilder.Warnings |= BackupCreationWarnings.CouldNotCompressWorldArchive;
+		}
+	}
+
+	private async Task<bool> CopyWorldAndCreateTarArchive(string backupFolderPath, string backupFilePath, BackupCreationResult.Builder resultBuilder) {
+		string temporaryFolderPath = Path.Combine(backupFolderPath, "temp");
+		
+		try {
+			if (!await CopyWorldToTemporaryFolder(temporaryFolderPath)) {
+				resultBuilder.Kind = BackupCreationResultKind.CouldNotCopyWorldToTemporaryFolder;
+				return false;
+			}
+
+			if (!await CreateTarArchive(temporaryFolderPath, backupFilePath)) {
+				resultBuilder.Kind = BackupCreationResultKind.CouldNotCreateWorldArchive;
+				return false;
+			}
+
+			return true;
+		} finally {
+			try {
+				Directory.Delete(temporaryFolderPath, recursive: true);
+			} catch (Exception e) {
+				resultBuilder.Warnings |= BackupCreationWarnings.CouldNotDeleteTemporaryFolder;
+				logger.Error(e, "Could not delete temporary world folder: {Folder}", temporaryFolderPath);
+			}
+		}
+	}
+
+	private async Task<bool> CopyWorldToTemporaryFolder(string temporaryFolderPath) {
+		try {
+			await CopyDirectory(new DirectoryInfo(instanceProperties.InstanceFolder), temporaryFolderPath, ImmutableList<string>.Empty);
+			return true;
+		} catch (Exception e) {
+			logger.Error(e, "Could not copy world to temporary folder.");
+			return false;
+		}
+	}
+
+	private async Task<bool> CreateTarArchive(string sourceFolderPath, string backupFilePath) {
+		try {
+			await TarFile.CreateFromDirectoryAsync(sourceFolderPath, backupFilePath, false, cancellationToken);
+			return true;
+		} catch (Exception e) {
+			logger.Error(e, "Could not create archive.");
+			return false;
+		}
+	}
+
+	private async Task CopyDirectory(DirectoryInfo sourceFolder, string destinationFolderPath, ImmutableList<string> relativePath) {
+		cancellationToken.ThrowIfCancellationRequested();
+
+		bool needsToCreateFolder = true;
+		
+		foreach (FileInfo file in sourceFolder.EnumerateFiles()) {
+			var filePath = relativePath.Add(file.Name);
+			if (IsFileSkipped(filePath)) {
+				logger.Verbose("Skipping file: {File}", string.Join('/', filePath));
+				continue;
+			}
+
+			if (needsToCreateFolder) {
+				needsToCreateFolder = false;
+				Directories.Create(destinationFolderPath, Chmod.URWX);
+			}
+
+			await CopyFileWithRetries(file, destinationFolderPath);
+		}
+
+		foreach (DirectoryInfo directory in sourceFolder.EnumerateDirectories()) {
+			var folderPath = relativePath.Add(directory.Name);
+			if (IsFolderSkipped(folderPath)) {
+				logger.Verbose("Skipping folder: {Folder}", string.Join('/', folderPath));
+				continue;
+			}
+			
+			await CopyDirectory(directory, Path.Join(destinationFolderPath, directory.Name), folderPath);
+		}
+	}
+
+	private async Task CopyFileWithRetries(FileInfo sourceFile, string destinationFolderPath) {
+		var destinationFilePath = Path.Combine(destinationFolderPath, sourceFile.Name);
+		
+		const int TotalAttempts = 10;
+		for (int attempt = 1; attempt <= TotalAttempts; attempt++) {
+			try {
+				sourceFile.CopyTo(destinationFilePath);
+				return;
+			} catch (IOException) {
+				if (attempt == TotalAttempts) {
+					throw;
+				}
+				else {
+					logger.Warning("Failed copying file {File}, retrying...", sourceFile.FullName);
+					await Task.Delay(200, cancellationToken);
+				}
+			}
+		}
+	}
+}

Agent/Phantom.Agent.Services/Backups/BackupCompressor.cs

@@ -0,0 +1,103 @@
+using System.Diagnostics;
+using Phantom.Common.Logging;
+using Serilog;
+
+namespace Phantom.Agent.Services.Backups; 
+
+static class BackupCompressor {
+	private static ILogger Logger { get; } = PhantomLogger.Create(nameof(BackupCompressor));
+	
+	private const int Quality = 10;
+	private const int Memory = 26;
+	private const int Threads = 3;
+	
+	public static async Task<string?> Compress(string sourceFilePath, CancellationToken cancellationToken) {
+		if (sourceFilePath.Contains('"')) {
+			Logger.Error("Could not compress backup, archive path contains quotes: {Path}", sourceFilePath);
+			return null;
+		}
+		
+		var destinationFilePath = sourceFilePath + ".zst";
+
+		if (!await TryCompressFile(sourceFilePath, destinationFilePath, cancellationToken)) {
+			try {
+				File.Delete(destinationFilePath);
+			} catch (Exception e) {
+				Logger.Error(e, "Could not delete compresed archive after unsuccessful compression: {Path}", destinationFilePath);
+			}
+
+			return null;
+		}
+
+		return destinationFilePath;
+	}
+
+	private static async Task<bool> TryCompressFile(string sourceFilePath, string destinationFilePath, CancellationToken cancellationToken) {
+		var workingDirectory = Path.GetDirectoryName(sourceFilePath);
+		if (string.IsNullOrEmpty(workingDirectory)) {
+			Logger.Error("Invalid destination path: {Path}", destinationFilePath);
+			return false;
+		}
+		
+		var startInfo = new ProcessStartInfo {
+			FileName = "zstd",
+			WorkingDirectory = workingDirectory,
+			Arguments = $"-{Quality} --long={Memory} -T{Threads} -c --rm --no-progress -c -o \"{destinationFilePath}\" -- \"{sourceFilePath}\"",
+			RedirectStandardOutput = true,
+			RedirectStandardError = true
+		};
+
+		using var process = new Process { StartInfo = startInfo };
+		process.OutputDataReceived += OnZstdProcessOutput;
+		process.ErrorDataReceived += OnZstdProcessOutput;
+
+		try {
+			process.Start();
+			process.BeginOutputReadLine();
+			process.BeginErrorReadLine();
+		} catch (Exception e) {
+			Logger.Error(e, "Caught exception launching zstd process.");
+		}
+
+		try {
+			await process.WaitForExitAsync(cancellationToken);
+		} catch (OperationCanceledException) {
+			await TryKillProcess(process);
+			return false;
+		} catch (Exception e) {
+			Logger.Error(e, "Caught exception waiting for zstd process to exit.");
+			return false;
+		}
+
+		if (!process.HasExited) {
+			await TryKillProcess(process);
+			return false;
+		}
+
+		if (process.ExitCode != 0) {
+			Logger.Error("Zstd process exited with code {ExitCode}.", process.ExitCode);
+			return false;
+		}
+
+		return true;
+	}
+
+	private static void OnZstdProcessOutput(object sender, DataReceivedEventArgs e) {
+		if (!string.IsNullOrWhiteSpace(e.Data)) {
+			Logger.Verbose("[Zstd] {Line}", e.Data);
+		}
+	}
+
+	private static async Task TryKillProcess(Process process) {
+		CancellationTokenSource timeout = new CancellationTokenSource(TimeSpan.FromSeconds(1));
+
+		try {
+			process.Kill();
+			await process.WaitForExitAsync(timeout.Token);
+		} catch (OperationCanceledException) {
+			Logger.Error("Timed out waiting for killed zstd process to exit.");
+		} catch (Exception e) {
+			Logger.Error(e, "Caught exception killing zstd process.");
+		}
+	}
+}

Agent/Phantom.Agent.Services/Backups/BackupManager.cs

@@ -0,0 +1,135 @@
+using System.Text.RegularExpressions;
+using Phantom.Agent.Minecraft.Command;
+using Phantom.Agent.Minecraft.Instance;
+using Phantom.Common.Data.Backups;
+using Phantom.Common.Logging;
+using Serilog;
+
+namespace Phantom.Agent.Services.Backups;
+
+sealed partial class BackupManager {
+	private readonly string basePath;
+
+	public BackupManager(AgentFolders agentFolders) {
+		this.basePath = agentFolders.BackupsFolderPath;
+	}
+
+	public async Task<BackupCreationResult> CreateBackup(string loggerName, InstanceSession session, CancellationToken cancellationToken) {
+		return await new BackupCreator(basePath, loggerName, session, cancellationToken).CreateBackup();
+	}
+
+	private sealed class BackupCreator {
+		private readonly string basePath;
+		private readonly string loggerName;
+		private readonly ILogger logger;
+		private readonly InstanceSession session;
+		private readonly BackupCommandListener listener;
+		private readonly CancellationToken cancellationToken;
+
+		public BackupCreator(string basePath, string loggerName, InstanceSession session, CancellationToken cancellationToken) {
+			this.basePath = basePath;
+			this.loggerName = loggerName;
+			this.logger = PhantomLogger.Create<BackupManager>(loggerName);
+			this.session = session;
+			this.listener = new BackupCommandListener(logger);
+			this.cancellationToken = cancellationToken;
+		}
+
+		public async Task<BackupCreationResult> CreateBackup() {
+			session.AddOutputListener(listener.OnOutput, 0);
+			try {
+				var resultBuilder = new BackupCreationResult.Builder();
+				await RunBackupProcess(resultBuilder);
+				return resultBuilder.Build();
+			} finally {
+				session.RemoveOutputListener(listener.OnOutput);
+			}
+		}
+		
+		private async Task RunBackupProcess(BackupCreationResult.Builder resultBuilder) {
+			try {
+				await DisableAutomaticSaving();
+				await SaveAllChunks();
+				await new BackupArchiver(loggerName, session.InstanceProperties, cancellationToken).ArchiveWorld(basePath, resultBuilder);
+			} catch (OperationCanceledException) {
+				resultBuilder.Kind = BackupCreationResultKind.BackupCancelled;
+				logger.Warning("Backup creation was cancelled.");
+			} catch (Exception e) {
+				resultBuilder.Kind = BackupCreationResultKind.UnknownError;
+				logger.Error(e, "Caught exception while creating an instance backup.");
+			} finally {
+				try {
+					await EnableAutomaticSaving();
+				} catch (OperationCanceledException) {
+					// ignore
+				} catch (Exception e) {
+					resultBuilder.Warnings |= BackupCreationWarnings.CouldNotRestoreAutomaticSaving;
+					logger.Error(e, "Caught exception while enabling automatic saving after creating an instance backup.");
+				}
+			}
+		}
+
+		private async Task DisableAutomaticSaving() {
+			await session.SendCommand(MinecraftCommand.SaveOff, cancellationToken);
+			await listener.AutomaticSavingDisabled.Task.WaitAsync(cancellationToken);
+		}
+
+		private async Task SaveAllChunks() {
+			// TODO Try if not flushing and waiting a few seconds before flushing reduces lag.
+			await session.SendCommand(MinecraftCommand.SaveAll(flush: true), cancellationToken);
+			await listener.SavedTheGame.Task.WaitAsync(cancellationToken);
+		}
+
+		private async Task EnableAutomaticSaving() {
+			await session.SendCommand(MinecraftCommand.SaveOn, cancellationToken);
+			await listener.AutomaticSavingEnabled.Task.WaitAsync(cancellationToken);
+		}
+	}
+	
+	private sealed partial class BackupCommandListener {
+		[GeneratedRegex(@"^\[(?:.*?)\] \[Server thread/INFO\]: (.*?)$", RegexOptions.NonBacktracking)]
+		private static partial Regex ServerThreadInfoRegex();
+
+		private readonly ILogger logger;
+
+		public BackupCommandListener(ILogger logger) {
+			this.logger = logger;
+		}
+
+		public TaskCompletionSource AutomaticSavingDisabled { get; } = new ();
+		public TaskCompletionSource SavedTheGame { get; } = new ();
+		public TaskCompletionSource AutomaticSavingEnabled { get; } = new ();
+
+		public void OnOutput(object? sender, string? line) {
+			if (line == null) {
+				return;
+			}
+
+			var match = ServerThreadInfoRegex().Match(line);
+			if (!match.Success) {
+				return;
+			}
+
+			string info = match.Groups[1].Value;
+
+			if (!AutomaticSavingDisabled.Task.IsCompleted) {
+				if (info == "Automatic saving is now disabled") {
+					logger.Verbose("Detected that automatic saving is disabled.");
+					AutomaticSavingDisabled.SetResult();
+				}
+			}
+			else if (!SavedTheGame.Task.IsCompleted) {
+				if (info == "Saved the game") {
+					logger.Verbose("Detected that the game is saved.");
+					SavedTheGame.SetResult();
+				}
+			}
+			else if (!AutomaticSavingEnabled.Task.IsCompleted) {
+				if (info == "Automatic saving is now enabled") {
+					logger.Verbose("Detected that automatic saving is enabled.");
+					AutomaticSavingEnabled.SetResult();
+				}
+			}
+		}
+	}
+}

Agent/Phantom.Agent.Services/Instances/InstanceSessionManager.cs

@@ -82,6 +82,7 @@ sealed class InstanceSessionManager : IDisposable {
 			);
 
 			var properties = new InstanceProperties(
+				instanceGuid,
 				configuration.JavaRuntimeGuid,
 				jvmProperties,
 				configuration.JvmArguments,

Common/Phantom.Common.Data/Backups/BackupCreationResult.cs

@@ -0,0 +1,18 @@
+using MemoryPack;
+
+namespace Phantom.Common.Data.Backups;
+
+[MemoryPackable]
+public sealed partial record BackupCreationResult(
+	[property: MemoryPackOrder(0)] BackupCreationResultKind Kind,
+	[property: MemoryPackOrder(1)] BackupCreationWarnings Warnings = BackupCreationWarnings.None
+) {
+	public sealed class Builder {
+		public BackupCreationResultKind Kind { get; set; } = BackupCreationResultKind.Success;
+		public BackupCreationWarnings Warnings { get; set; }
+		
+		public BackupCreationResult Build() {
+			return new BackupCreationResult(Kind, Warnings);
+		}
+	}
+}

Common/Phantom.Common.Data/Backups/BackupCreationResultKind.cs

@@ -0,0 +1,25 @@
+namespace Phantom.Common.Data.Backups;
+
+public enum BackupCreationResultKind : byte {
+	UnknownError,
+	Success,
+	BackupCancelled,
+	BackupAlreadyExists,
+	CouldNotCreateBackupFolder,
+	CouldNotCopyWorldToTemporaryFolder,
+	CouldNotCreateWorldArchive
+}
+
+public static class BackupCreationResultSummaryExtensions {
+	public static string ToSentence(this BackupCreationResultKind kind) {
+		return kind switch {
+			BackupCreationResultKind.Success                            => "Backup created successfully.",
+			BackupCreationResultKind.BackupCancelled                    => "Backup cancelled.",
+			BackupCreationResultKind.BackupAlreadyExists                => "Backup with the same name already exists.",
+			BackupCreationResultKind.CouldNotCreateBackupFolder         => "Could not create backup folder.",
+			BackupCreationResultKind.CouldNotCopyWorldToTemporaryFolder => "Could not copy world to temporary folder.",
+			BackupCreationResultKind.CouldNotCreateWorldArchive         => "Could not create world archive.",
+			_                                                              => "Unknown error."
+		};
+	}
+}

Common/Phantom.Common.Data/Backups/BackupCreationWarnings.cs

@@ -0,0 +1,9 @@
+namespace Phantom.Common.Data.Backups; 
+
+[Flags]
+public enum BackupCreationWarnings : byte {
+	None = 0,
+	CouldNotDeleteTemporaryFolder = 1 << 0,
+	CouldNotCompressWorldArchive = 1 << 1,
+	CouldNotRestoreAutomaticSaving = 1 << 2
+}