Mark Agents as disconnected if the Server does not receive keep-alive messages for too long

Server/Phantom.Server.Services/Agents/Agent.cs

@@ -15,12 +15,17 @@ public sealed record Agent(
 ) {
 	internal AgentConnection? Connection { get; init; }
 	
+	public bool IsOnline { get; internal init; }
+	public bool IsOffline => !IsOnline;
+
 	internal Agent(AgentInfo info) : this(info.Guid, info.Name, info.Version, info.MaxInstances, info.MaxMemory, info.AllowedServerPorts, info.AllowedRconPorts) {}
 
-	public bool IsOnline => Connection is not null;
-	public bool IsOffline => Connection is null;
+	internal Agent AsDisconnected() => this with {
+		IsOnline = false
+	};
 	
 	internal Agent AsOffline() => this with {
-		Connection = null
+		Connection = null,
+		IsOnline = false
 	};
 }

Server/Phantom.Server.Services/Agents/AgentManager.cs

@@ -10,6 +10,7 @@ using Phantom.Server.Services.Instances;
 using Phantom.Server.Services.Rpc;
 using Phantom.Utils.Collections;
 using Phantom.Utils.Events;
+using Phantom.Utils.Threading;
 using Serilog;
 
 namespace Phantom.Server.Services.Agents;
@@ -17,6 +18,9 @@ namespace Phantom.Server.Services.Agents;
 public sealed class AgentManager {
 	private static readonly ILogger Logger = PhantomLogger.Create<AgentManager>();
 
+	private static readonly TimeSpan DisconnectionRecheckInterval = TimeSpan.FromSeconds(5);
+	private static readonly TimeSpan DisconnectionThreshold = TimeSpan.FromSeconds(12);
+
 	private readonly ObservableAgents agents = new (PhantomLogger.Create<AgentManager, ObservableAgents>());
 
 	public EventSubscribers<ImmutableArray<Agent>> AgentsChanged => agents.Subs;
@@ -25,10 +29,11 @@ public sealed class AgentManager {
 	private readonly AgentAuthToken authToken;
 	private readonly DatabaseProvider databaseProvider;
 
-	public AgentManager(ServiceConfiguration configuration, AgentAuthToken authToken, DatabaseProvider databaseProvider) {
+	public AgentManager(ServiceConfiguration configuration, AgentAuthToken authToken, DatabaseProvider databaseProvider, TaskManager taskManager) {
 		this.cancellationToken = configuration.CancellationToken;
 		this.authToken = authToken;
 		this.databaseProvider = databaseProvider;
+		taskManager.Run(RefreshAgentStatus);
 	}
 
 	public async Task Initialize() {
@@ -51,6 +56,7 @@ public sealed class AgentManager {
 
 		var agent = new Agent(agentInfo) {
 			LastPing = DateTimeOffset.Now,
+			IsOnline = true,
 			Connection = new AgentConnection(connection)
 		};
 
@@ -58,12 +64,12 @@ public sealed class AgentManager {
 
 		using (var scope = databaseProvider.CreateScope()) {
 			var entity = scope.Ctx.AgentUpsert.Fetch(agent.Guid);
-			
+
 			entity.Name = agent.Name;
 			entity.Version = agent.Version;
 			entity.MaxInstances = agent.MaxInstances;
 			entity.MaxMemory = agent.MaxMemory;
-			
+
 			await scope.Ctx.SaveChangesAsync(cancellationToken);
 		}
 
@@ -92,6 +98,20 @@ public sealed class AgentManager {
 		agents.Update(agentGuid, static agent => agent with { LastPing = DateTimeOffset.Now });
 	}
 
+	private async Task RefreshAgentStatus() {
+		static Agent MarkAgentAsOffline(Agent agent) {
+			Logger.Warning("Lost connection to agent \"{Name}\" (GUID {Guid}).", agent.Name, agent.Guid);
+			return agent.AsDisconnected();
+		}
+
+		while (!cancellationToken.IsCancellationRequested) {
+			await Task.Delay(DisconnectionRecheckInterval, cancellationToken);
+
+			var now = DateTimeOffset.Now;
+			agents.UpdateAllIf(MarkAgentAsOffline, agent => agent.IsOnline && agent.LastPing is {} lastPing && now - lastPing >= DisconnectionThreshold);
+		}
+	}
+
 	private async Task<bool> SendMessage<TMessage>(Guid guid, TMessage message) where TMessage : IMessageToAgent {
 		var connection = agents.GetConnection(guid);
 		if (connection != null) {
@@ -137,6 +157,10 @@ public sealed class AgentManager {
 			UpdateIf(agents.TryReplace(guid, updater));
 		}
 
+		public void UpdateAllIf(Func<Agent, Agent> updater, Predicate<Agent> predicate) {
+			UpdateIf(agents.TryReplaceAllIf(updater, predicate));
+		}
+
 		public bool TryUnregister(Guid guid, RpcClientConnection connection) {
 			return UpdateIf(agents.TryReplaceIf(guid, static oldAgent => oldAgent.AsOffline(), oldAgent => oldAgent.Connection?.IsSame(connection) == true));
 		}