Stream messages to viewer using JSON lines

app/Resources/Viewer/scripts/bootstrap.mjs

@@ -36,18 +36,69 @@ document.addEventListener("DOMContentLoaded", () => {
 		gui.scrollMessagesToTop();
 	});
 	
+	async function fetchUrl(path, contentType) {
+		const response = await fetch("/" + path + "?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), {
+			method: "GET",
+			headers: {
+				"Content-Type": contentType,
+			},
+			credentials: "omit",
+			redirect: "error",
+		});
+		
+		if (!response.ok) {
+			throw "Unexpected response status: " + response.statusText;
+		}
+		
+		return response;
+	}
+	
+	async function processLines(response, callback) {
+		let body = "";
+		
+		for await (const chunk of response.body.pipeThrough(new TextDecoderStream("utf-8"))) {
+			body += chunk;
+			
+			let startIndex = 0;
+			
+			while (true) {
+				const endIndex = body.indexOf("\n", startIndex);
+				if (endIndex === -1) {
+					break;
+				}
+				
+				callback(body.substring(startIndex, endIndex));
+				startIndex = endIndex + 1;
+			}
+			
+			body = body.substring(startIndex);
+		}
+		
+		if (body !== "") {
+			callback(body);
+		}
+	}
+	
 	async function loadData() {
 		try {
-			const response = await fetch("/get-viewer-data?token=" + encodeURIComponent(window.DHT_SERVER_TOKEN) + "&session=" + encodeURIComponent(window.DHT_SERVER_SESSION), {
+			const metadataResponse = await fetchUrl("get-viewer-metadata", "application/json");
-				method: "GET",
+			const metadataJson = await metadataResponse.json();
-				headers: {
+			
-					"Content-Type": "application/json",
+			const messagesResponse = await fetchUrl("get-viewer-messages", "application/x-ndjson");
-				},
+			const messages = {};
-				credentials: "omit",
+			
-				redirect: "error",
+			await processLines(messagesResponse, line => {
+				const message = JSON.parse(line);
+				const channel = message.c;
+				
+				const channelMessages = messages[channel] || (messages[channel] = {});
+				channelMessages[message.id] = message;
+				
+				delete message.id;
+				delete message.c;
 			});
-
+			
-			state.uploadFile(await response.json());
+			state.uploadFile(metadataJson, messages);
 		} catch (e) {
 			console.error(e);
 			alert("Could not load data, see console for details.");

app/Resources/Viewer/scripts/state.mjs

@@ -6,8 +6,7 @@ export default (function() {
 	/**
 	 * @type {{}}
 	 * @property {{}} users
-	 * @property {String[]} userindex
+	 * @property {{}} servers
-	 * @property {{}[]} servers
 	 * @property {{}} channels
 	 */
 	let loadedFileMeta;
@@ -20,20 +19,16 @@ export default (function() {
 	let currentPage;
 	let messagesPerPage;
 	
-	const getUser = function(index) {
+	const getUser = function(id) {
-		return loadedFileMeta.users[loadedFileMeta.userindex[index]] || { "name": "<unknown>" };
+		return loadedFileMeta.users[id] || { "name": "<unknown>" };
-	};
-	
-	const getUserId = function(index) {
-		return loadedFileMeta.userindex[index];
 	};
 	
 	const getUserList = function() {
 		return loadedFileMeta ? loadedFileMeta.users : [];
 	};
 	
-	const getServer = function(index) {
+	const getServer = function(id) {
-		return loadedFileMeta.servers[index] || { "name": "<unknown>", "type": "unknown" };
+		return loadedFileMeta.servers[id] || { "name": "<unknown>", "type": "unknown" };
 	};
 	
 	const generateChannelHierarchy = function() {
@@ -207,7 +202,7 @@ export default (function() {
 			 */
 			const message = messages[key];
 			const user = getUser(message.u);
-			const avatar = user.avatar ? { id: getUserId(message.u), path: user.avatar } : null;
+			const avatar = user.avatar ? { id: message.u, path: user.avatar } : null;
 			
 			const obj = {
 				user,
@@ -235,7 +230,7 @@ export default (function() {
 			if ("r" in message) {
 				const replyMessage = getMessageById(message.r);
 				const replyUser = replyMessage ? getUser(replyMessage.u) : null;
-				const replyAvatar = replyUser && replyUser.avatar ? { id: getUserId(replyMessage.u), path: replyUser.avatar } : null;
+				const replyAvatar = replyUser && replyUser.avatar ? { id: replyMessage.u, path: replyUser.avatar } : null;
 				
 				obj["reply"] = replyMessage ? {
 					"id": message.r,
@@ -293,20 +288,17 @@ export default (function() {
 			eventOnUsersRefreshed = callback;
 		},
 		
-		/**
+		uploadFile(meta, data) {
-		 * @param {{ meta, data }} file
-		 */
-		uploadFile(file) {
 			if (loadedFileMeta != null) {
 				throw "A file is already loaded!";
 			}
 			
-			if (!file || typeof file.meta !== "object" || typeof file.data !== "object") {
+			if (typeof meta !== "object" || typeof data !== "object") {
 				throw "Invalid file format!";
 			}
 			
-			loadedFileMeta = file.meta;
+			loadedFileMeta = meta;
-			loadedFileData = file.data;
+			loadedFileData = data;
 			loadedMessages = null;
 			
 			selectedChannel = null;

app/Server/Database/Export/Snowflake.cs

@@ -1,3 +1,5 @@
 namespace DHT.Server.Database.Export;
 
-readonly record struct Snowflake(ulong Id);
+readonly record struct Snowflake(ulong Id) {
+	public static implicit operator Snowflake(ulong id) => new (id);
+}

app/Server/Database/Export/ViewerJson.cs

@@ -3,14 +3,10 @@ using System.Text.Json.Serialization;
 
 namespace DHT.Server.Database.Export;
 
-sealed class ViewerJson {
+static class ViewerJson {
-	public required JsonMeta Meta { get; init; }
-	public required Dictionary<Snowflake, Dictionary<Snowflake, JsonMessage>> Data { get; init; }
-
 	public sealed class JsonMeta {
 		public required Dictionary<Snowflake, JsonUser> Users { get; init; }
-		public required List<Snowflake> Userindex { get; init; }
+		public required Dictionary<Snowflake, JsonServer> Servers { get; init; }
-		public required List<JsonServer> Servers { get; init; }
 		public required Dictionary<Snowflake, JsonChannel> Channels { get; init; }
 	}
 	
@@ -30,7 +26,7 @@ sealed class ViewerJson {
 	}
 
 	public sealed class JsonChannel {
-		public required int Server { get; init; }
+		public required Snowflake Server { get; init; }
 		public required string Name { get; init; }
 		
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
@@ -47,7 +43,9 @@ sealed class ViewerJson {
 	}
 
 	public sealed class JsonMessage {
-		public required int U { get; init; }
+		public required Snowflake Id { get; init; }
+		public required Snowflake C { get; init; }
+		public required Snowflake U { get; init; }
 		public required long T { get; init; }
 
 		[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]

app/Server/Database/Export/ViewerJsonExport.cs

@@ -2,7 +2,9 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
@@ -13,106 +15,90 @@ namespace DHT.Server.Database.Export;
 static class ViewerJsonExport {
 	private static readonly Log Log = Log.ForType(typeof(ViewerJsonExport));
 
-	public static async Task Generate(Stream stream, IDatabaseFile db, MessageFilter? filter = null) {
+	public static async Task GetMetadata(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
 		var perf = Log.Start();
 
-		var includedUserIds = new HashSet<ulong>();
+		var includedChannels = new List<Channel>();
-		var includedChannelIds = new HashSet<ulong>();
 		var includedServerIds = new HashSet<ulong>();
 
-		var includedMessages = await db.Messages.Get(filter).ToListAsync();
+		var channelIdFilter = filter?.ChannelIds;
-		var includedChannels = new List<Channel>();
 
-		foreach (var message in includedMessages) {
+		await foreach (var channel in db.Channels.Get(cancellationToken)) {
-			includedUserIds.Add(message.Sender);
+			if (channelIdFilter == null || channelIdFilter.Contains(channel.Id)) {
-			includedChannelIds.Add(message.Channel);
-		}
-
-		await foreach (var channel in db.Channels.Get()) {
-			if (includedChannelIds.Contains(channel.Id)) {
 				includedChannels.Add(channel);
 				includedServerIds.Add(channel.Server);
 			}
 		}
 
-		var (users, userIndex, userIndices) = await GenerateUserList(db, includedUserIds);
+		var users = await GenerateUserList(db, cancellationToken);
-		var (servers, serverIndices) = await GenerateServerList(db, includedServerIds);
+		var servers = await GenerateServerList(db, includedServerIds, cancellationToken);
-		var channels = GenerateChannelList(includedChannels, serverIndices);
+		var channels = GenerateChannelList(includedChannels);
+
+		var meta = new ViewerJson.JsonMeta {
+			Users = users,
+			Servers = servers,
+			Channels = channels
+		};
 
 		perf.Step("Collect database data");
 
-		var value = new ViewerJson {
+		await JsonSerializer.SerializeAsync(stream, meta, ViewerJsonMetadataContext.Default.JsonMeta, cancellationToken);
-			Meta = new ViewerJson.JsonMeta {
-				Users = users,
-				Userindex = userIndex,
-				Servers = servers,
-				Channels = channels
-			},
-			Data = GenerateMessageList(includedMessages, userIndices)
-		};
-
-		perf.Step("Generate value object");
-
-		await JsonSerializer.SerializeAsync(stream, value, ViewerJsonContext.Default.ViewerJson);
 
 		perf.Step("Serialize to JSON");
 		perf.End();
 	}
 
-	private static async Task<(Dictionary<Snowflake, ViewerJson.JsonUser> Users, List<Snowflake> UserIndex, Dictionary<ulong, int> UserIndices)> GenerateUserList(IDatabaseFile db, HashSet<ulong> userIds) {
+	public static async Task GetMessages(Stream stream, IDatabaseFile db, MessageFilter? filter = null, CancellationToken cancellationToken = default) {
+		var perf = Log.Start();
+
+		ReadOnlyMemory<byte> newLine = "\n"u8.ToArray();
+		
+		await foreach(var message in GenerateMessageList(db, filter, cancellationToken)) {
+			await JsonSerializer.SerializeAsync(stream, message, ViewerJsonMessageContext.Default.JsonMessage, cancellationToken);
+			await stream.WriteAsync(newLine, cancellationToken);
+		}
+
+		perf.Step("Generate and serialize messages to JSON");
+		perf.End();
+	}
+
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonUser>> GenerateUserList(IDatabaseFile db, CancellationToken cancellationToken) {
 		var users = new Dictionary<Snowflake, ViewerJson.JsonUser>();
-		var userIndex = new List<Snowflake>();
-		var userIndices = new Dictionary<ulong, int>();
 
-		await foreach (var user in db.Users.Get()) {
+		await foreach (var user in db.Users.Get(cancellationToken)) {
-			var id = user.Id;
+			users[user.Id] = new ViewerJson.JsonUser {
-			if (!userIds.Contains(id)) {
-				continue;
-			}
-
-			var idSnowflake = new Snowflake(id);
-			userIndices[id] = users.Count;
-			userIndex.Add(idSnowflake);
-			
-			users[idSnowflake] = new ViewerJson.JsonUser {
 				Name = user.Name,
 				Avatar = user.AvatarUrl,
 				Tag = user.Discriminator
 			};
 		}
 
-		return (users, userIndex, userIndices);
+		return users;
 	}
 
-	private static async Task<(List<ViewerJson.JsonServer> Servers, Dictionary<ulong, int> ServerIndices)> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds) {
+	private static async Task<Dictionary<Snowflake, ViewerJson.JsonServer>> GenerateServerList(IDatabaseFile db, HashSet<ulong> serverIds, CancellationToken cancellationToken) {
-		var servers = new List<ViewerJson.JsonServer>();
+		var servers = new Dictionary<Snowflake, ViewerJson.JsonServer>();
-		var serverIndices = new Dictionary<ulong, int>();
 
-		await foreach (var server in db.Servers.Get()) {
+		await foreach (var server in db.Servers.Get(cancellationToken)) {
-			var id = server.Id;
+			if (!serverIds.Contains(server.Id)) {
-			if (!serverIds.Contains(id)) {
 				continue;
 			}
 
-			serverIndices[id] = servers.Count;
+			servers[server.Id] = new ViewerJson.JsonServer {
-			
-			servers.Add(new ViewerJson.JsonServer {
 				Name = server.Name,
 				Type = ServerTypes.ToJsonViewerString(server.Type)
-			});
+			};
 		}
 
-		return (servers, serverIndices);
+		return servers;
 	}
 
-	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels, Dictionary<ulong, int> serverIndices) {
+	private static Dictionary<Snowflake, ViewerJson.JsonChannel> GenerateChannelList(List<Channel> includedChannels) {
 		var channels = new Dictionary<Snowflake, ViewerJson.JsonChannel>();
 
 		foreach (var channel in includedChannels) {
-			var channelIdSnowflake = new Snowflake(channel.Id);
+			channels[channel.Id] = new ViewerJson.JsonChannel {
-			
+				Server = channel.Server,
-			channels[channelIdSnowflake] = new ViewerJson.JsonChannel {
-				Server = serverIndices[channel.Server],
 				Name = channel.Name,
 				Parent = channel.ParentId?.ToString(),
 				Position = channel.Position,
@@ -124,51 +110,40 @@ static class ViewerJsonExport {
 		return channels;
 	}
 
-	private static Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>> GenerateMessageList(List<Message> includedMessages, Dictionary<ulong, int> userIndices) {
+	private static async IAsyncEnumerable<ViewerJson.JsonMessage> GenerateMessageList(IDatabaseFile db, MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
-		var data = new Dictionary<Snowflake, Dictionary<Snowflake, ViewerJson.JsonMessage>>();
+		await foreach (var message in db.Messages.Get(filter, cancellationToken)) {
+			yield return new ViewerJson.JsonMessage {
+				Id = message.Id,
+				C = message.Channel,
+				U = message.Sender,
+				T = message.Timestamp,
+				M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
+				Te = message.EditTimestamp,
+				R = message.RepliedToId?.ToString(),
 
-		foreach (var grouping in includedMessages.GroupBy(static message => message.Channel)) {
+				A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => {
-			var channelIdSnowflake = new Snowflake(grouping.Key);
+					var a = new ViewerJson.JsonMessageAttachment {
-			var channelData = new Dictionary<Snowflake, ViewerJson.JsonMessage>();
+						Url = attachment.DownloadUrl,
+						Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl
+					};
 
-			foreach (var message in grouping) {
+					if (attachment is { Width: not null, Height: not null }) {
-				var messageIdSnowflake = new Snowflake(message.Id);
+						a.Width = attachment.Width;
-				
+						a.Height = attachment.Height;
-				channelData[messageIdSnowflake] = new ViewerJson.JsonMessage {
+					}
-					U = userIndices[message.Sender],
-					T = message.Timestamp,
-					M = string.IsNullOrEmpty(message.Text) ? null : message.Text,
-					Te = message.EditTimestamp,
-					R = message.RepliedToId?.ToString(),
-					
-					A = message.Attachments.IsEmpty ? null : message.Attachments.Select(static attachment => {
-						var a = new ViewerJson.JsonMessageAttachment {
-							Url = attachment.DownloadUrl,
-							Name = Uri.TryCreate(attachment.NormalizedUrl, UriKind.Absolute, out var uri) ? Path.GetFileName(uri.LocalPath) : attachment.NormalizedUrl
-						};
 
-						if (attachment is { Width: not null, Height: not null }) {
+					return a;
-							a.Width = attachment.Width;
+				}).ToArray(),
-							a.Height = attachment.Height;
-						}
 
-						return a;
+				E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(),
-					}).ToArray(),
-					
-					E = message.Embeds.IsEmpty ? null : message.Embeds.Select(static embed => embed.Json).ToArray(),
-					
-					Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction {
-						Id = reaction.EmojiId?.ToString(),
-						N = reaction.EmojiName,
-						A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated),
-						C = reaction.Count
-					}).ToArray()
-				};
-			}
 
-			data[channelIdSnowflake] = channelData;
+				Re = message.Reactions.IsEmpty ? null : message.Reactions.Select(static reaction => new ViewerJson.JsonMessageReaction {
+					Id = reaction.EmojiId?.ToString(),
+					N = reaction.EmojiName,
+					A = reaction.EmojiFlags.HasFlag(EmojiFlags.Animated),
+					C = reaction.Count
+				}).ToArray()
+			};
 		}
-
-		return data;
 	}
 }

app/Server/Database/Export/ViewerJsonMessageContext.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace DHT.Server.Database.Export;
+
+[JsonSourceGenerationOptions(
+	Converters = [typeof(SnowflakeJsonSerializer)],
+	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
+	GenerationMode = JsonSourceGenerationMode.Default
+)]
+[JsonSerializable(typeof(ViewerJson.JsonMessage))]
+sealed partial class ViewerJsonMessageContext : JsonSerializerContext;

app/Server/Database/Export/ViewerJsonMetadataContext.cs

@@ -7,5 +7,5 @@ namespace DHT.Server.Database.Export;
 	PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
 	GenerationMode = JsonSourceGenerationMode.Default
 )]
-[JsonSerializable(typeof(ViewerJson))]
+[JsonSerializable(typeof(ViewerJson.JsonMeta))]
-sealed partial class ViewerJsonContext : JsonSerializerContext;
+sealed partial class ViewerJsonMetadataContext : JsonSerializerContext;

app/Server/Database/Repositories/IChannelRepository.cs

@@ -15,7 +15,7 @@ public interface IChannelRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Channel> Get();
+	IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IChannelRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -28,7 +28,7 @@ public interface IChannelRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Channel> Get() {
+		public IAsyncEnumerable<Channel> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Channel>();
 		}
 	}

app/Server/Database/Repositories/IDownloadRepository.cs

@@ -24,7 +24,7 @@ public interface IDownloadRepository {
 
 	Task<bool> GetDownloadData(string normalizedUrl, Func<Stream, Task> dataProcessor);
 	
-	Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor);
+	Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken = default);
 
 	IAsyncEnumerable<DownloadItem> PullPendingDownloadItems(int count, DownloadItemFilter filter, CancellationToken cancellationToken = default);
 	
@@ -55,7 +55,7 @@ public interface IDownloadRepository {
 			return Task.FromResult(false);
 		}
 
-		public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) {
+		public Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken) {
 			return Task.FromResult(false);
 		}
 

app/Server/Database/Repositories/IMessageRepository.cs

@@ -16,7 +16,7 @@ public interface IMessageRepository {
 	
 	Task<long> Count(MessageFilter? filter = null, CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Message> Get(MessageFilter? filter = null);
+	IAsyncEnumerable<Message> Get(MessageFilter? filter = null, CancellationToken cancellationToken = default);
 	
 	IAsyncEnumerable<ulong> GetIds(MessageFilter? filter = null);
 	
@@ -33,7 +33,7 @@ public interface IMessageRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Message> Get(MessageFilter? filter) {
+		public IAsyncEnumerable<Message> Get(MessageFilter? filter, CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Message>();
 		}
 

app/Server/Database/Repositories/IServerRepository.cs

@@ -14,7 +14,7 @@ public interface IServerRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<Data.Server> Get();
+	IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IServerRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -27,7 +27,7 @@ public interface IServerRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<Data.Server> Get() {
+		public IAsyncEnumerable<Data.Server> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<Data.Server>();
 		}
 	}

app/Server/Database/Repositories/IUserRepository.cs

@@ -15,7 +15,7 @@ public interface IUserRepository {
 	
 	Task<long> Count(CancellationToken cancellationToken = default);
 	
-	IAsyncEnumerable<User> Get();
+	IAsyncEnumerable<User> Get(CancellationToken cancellationToken = default);
 
 	internal sealed class Dummy : IUserRepository {
 		public IObservable<long> TotalCount { get; } = Observable.Return(0L);
@@ -28,7 +28,7 @@ public interface IUserRepository {
 			return Task.FromResult(0L);
 		}
 
-		public IAsyncEnumerable<User> Get() {
+		public IAsyncEnumerable<User> Get(CancellationToken cancellationToken) {
 			return AsyncEnumerable.Empty<User>();
 		}
 	}

app/Server/Database/Sqlite/Repositories/SqliteChannelRepository.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -54,13 +55,13 @@ sealed class SqliteChannelRepository : BaseSqliteRepository, IChannelRepository
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM channels", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<Channel> Get() {
+	public async IAsyncEnumerable<Channel> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, server, name, parent_id, position, topic, nsfw FROM channels");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new Channel {
 				Id = reader.GetUint64(0),
 				Server = reader.GetUint64(1),

app/Server/Database/Sqlite/Repositories/SqliteDownloadRepository.cs

@@ -210,7 +210,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		return true;
 	}
 
-	public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, Task> dataProcessor) {
+	public async Task<bool> GetSuccessfulDownloadWithData(string normalizedUrl, Func<Data.Download, Stream, CancellationToken, Task> dataProcessor, CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command(
@@ -228,8 +228,8 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		string? type;
 		long rowid;
 		
-		await using (var reader = await cmd.ExecuteReaderAsync()) {
+		await using (var reader = await cmd.ExecuteReaderAsync(cancellationToken)) {
-			if (!await reader.ReadAsync()) {
+			if (!await reader.ReadAsync(cancellationToken)) {
 				return false;
 			}
 
@@ -239,7 +239,7 @@ sealed class SqliteDownloadRepository(SqliteConnectionPool pool) : BaseSqliteRep
 		}
 		
 		await using (var blob = new SqliteBlob(conn.InnerConnection, "download_blobs", "blob", rowid, readOnly: true)) {
-			await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob);
+			await dataProcessor(new Data.Download(normalizedUrl, downloadUrl, DownloadStatus.Success, type, (ulong) blob.Length), blob, cancellationToken);
 		}
 
 		return true;

app/Server/Database/Sqlite/Repositories/SqliteMessageRepository.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -213,7 +214,7 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 		}
 	}
 
-	public async IAsyncEnumerable<Message> Get(MessageFilter? filter) {
+	public async IAsyncEnumerable<Message> Get(MessageFilter? filter, [EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		const string AttachmentSql =
@@ -269,9 +270,9 @@ sealed class SqliteMessageRepository : BaseSqliteRepository, IMessageRepository
 			 """
 		);
 
-		await using var reader = await messageCmd.ExecuteReaderAsync();
+		await using var reader = await messageCmd.ExecuteReaderAsync(cancellationToken);
 
-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			ulong messageId = reader.GetUint64(0);
 
 			yield return new Message {

app/Server/Database/Sqlite/Repositories/SqliteServerRepository.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -46,13 +47,13 @@ sealed class SqliteServerRepository : BaseSqliteRepository, IServerRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM servers", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<Data.Server> Get() {
+	public async IAsyncEnumerable<Data.Server> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, name, type FROM servers");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new Data.Server {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),

app/Server/Database/Sqlite/Repositories/SqliteUserRepository.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
@@ -58,13 +59,13 @@ sealed class SqliteUserRepository : BaseSqliteRepository, IUserRepository {
 		return await conn.ExecuteReaderAsync("SELECT COUNT(*) FROM users", static reader => reader?.GetInt64(0) ?? 0L, cancellationToken);
 	}
 
-	public async IAsyncEnumerable<User> Get() {
+	public async IAsyncEnumerable<User> Get([EnumeratorCancellation] CancellationToken cancellationToken) {
 		await using var conn = await pool.Take();
 
 		await using var cmd = conn.Command("SELECT id, name, avatar_url, discriminator FROM users");
-		await using var reader = await cmd.ExecuteReaderAsync();
+		await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
 
-		while (await reader.ReadAsync()) {
+		while (await reader.ReadAsync(cancellationToken)) {
 			yield return new User {
 				Id = reader.GetUint64(0),
 				Name = reader.GetString(1),

app/Server/Endpoints/BaseEndpoint.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Database;
 using DHT.Utils.Http;
@@ -19,7 +20,9 @@ abstract class BaseEndpoint(IDatabaseFile db) {
 
 		try {
 			response.StatusCode = (int) HttpStatusCode.OK;
-			await Respond(ctx.Request, response);
+			await Respond(ctx.Request, response, ctx.RequestAborted);
+		} catch (OperationCanceledException) {
+			throw;
 		} catch (HttpException e) {
 			Log.Error(e);
 			response.StatusCode = (int) e.StatusCode;
@@ -35,7 +38,7 @@ abstract class BaseEndpoint(IDatabaseFile db) {
 		}
 	}
 
-	protected abstract Task Respond(HttpRequest request, HttpResponse response);
+	protected abstract Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken);
 
 	protected static async Task<JsonElement> ReadJson(HttpRequest request) {
 		try {
@@ -44,4 +47,13 @@ abstract class BaseEndpoint(IDatabaseFile db) {
 			throw new HttpException(HttpStatusCode.UnsupportedMediaType, "This endpoint only accepts JSON.");
 		}
 	}
+	
+	protected static Guid GetSessionId(HttpRequest request) {
+		if (request.Query.TryGetValue("session", out var sessionIdValue) && sessionIdValue.Count == 1 && Guid.TryParse(sessionIdValue[0], out Guid sessionId)) {
+			return sessionId;
+		}
+		else {
+			throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID.");
+		}
+	}
 }

app/Server/Endpoints/GetDownloadedFileEndpoint.cs

@@ -1,4 +1,7 @@
+using System;
+using System.IO;
 using System.Net;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Database;
 using DHT.Server.Download;
@@ -8,12 +11,16 @@ using Microsoft.AspNetCore.Http;
 namespace DHT.Server.Endpoints;
 
 sealed class GetDownloadedFileEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		string url = WebUtility.UrlDecode((string) request.RouteValues["url"]!);
 		string normalizedUrl = DiscordCdn.NormalizeUrl(url);
 		
-		if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, (download, stream) => response.WriteStreamAsync(download.Type, download.Size, stream))) {
+		if (!await Db.Downloads.GetSuccessfulDownloadWithData(normalizedUrl, WriteDataTo(response), cancellationToken)) {
 			response.Redirect(url, permanent: false);
 		}
 	}
+	
+	private static Func<Data.Download, Stream, CancellationToken, Task> WriteDataTo(HttpResponse response) {
+		return (download, stream, cancellationToken) => response.WriteStreamAsync(download.Type, download.Size, stream, cancellationToken);
+	}
 }

app/Server/Endpoints/GetTrackingScriptEndpoint.cs

@@ -1,4 +1,5 @@
 using System.Net.Mime;
+using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
 using DHT.Server.Database;
@@ -10,7 +11,7 @@ using Microsoft.AspNetCore.Http;
 namespace DHT.Server.Endpoints;
 
 sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parameters, ResourceLoader resources) : BaseEndpoint(db) {
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		string bootstrap = await resources.ReadTextAsync("Tracker/bootstrap.js");
 		string script = bootstrap.Replace("= 0; /*[PORT]*/", "= " + parameters.Port + ";")
 		                         .Replace("/*[TOKEN]*/", HttpUtility.JavaScriptStringEncode(parameters.Token))
@@ -20,6 +21,6 @@ sealed class GetTrackingScriptEndpoint(IDatabaseFile db, ServerParameters parame
 		                         .Replace("/*[DEBUGGER]*/", request.Query.ContainsKey("debug") ? "debugger;" : "");
 		
 		response.Headers.Append("X-DHT", "1");
-		await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script);
+		await response.WriteTextAsync(MediaTypeNames.Text.JavaScript, script, cancellationToken);
 	}
 }

app/Server/Endpoints/GetViewerDataEndpoint.cs

@@ -1,24 +0,0 @@
-using System;
-using System.Net;
-using System.Net.Mime;
-using System.Threading.Tasks;
-using DHT.Server.Database;
-using DHT.Server.Database.Export;
-using DHT.Server.Service.Viewer;
-using DHT.Utils.Http;
-using Microsoft.AspNetCore.Http;
-
-namespace DHT.Server.Endpoints; 
-
-sealed class GetViewerDataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
-		if (!request.Query.TryGetValue("session", out var sessionIdValue) || sessionIdValue.Count != 1 || !Guid.TryParse(sessionIdValue[0], out Guid sessionId)) {
-			throw new HttpException(HttpStatusCode.BadRequest, "Invalid session ID.");
-		}
-		
-		response.ContentType = MediaTypeNames.Application.Json;
-		
-		var session = viewerSessions.Get(sessionId);
-		await ViewerJsonExport.Generate(response.Body, Db, session.MessageFilter);
-	}
-}

app/Server/Endpoints/GetViewerMessagesEndpoint.cs

@@ -0,0 +1,18 @@
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMessagesEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = "application/x-ndjson";
+		return ViewerJsonExport.GetMessages(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}

app/Server/Endpoints/GetViewerMetadataEndpoint.cs

@@ -0,0 +1,19 @@
+using System.Net.Mime;
+using System.Threading;
+using System.Threading.Tasks;
+using DHT.Server.Database;
+using DHT.Server.Database.Export;
+using DHT.Server.Service.Viewer;
+using Microsoft.AspNetCore.Http;
+
+namespace DHT.Server.Endpoints; 
+
+sealed class GetViewerMetadataEndpoint(IDatabaseFile db, ViewerSessions viewerSessions) : BaseEndpoint(db) {
+	protected override Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
+		var sessionId = GetSessionId(request);
+		var session = viewerSessions.Get(sessionId);
+		
+		response.ContentType = MediaTypeNames.Application.Json;
+		return ViewerJsonExport.GetMetadata(response.Body, Db, session.MessageFilter, cancellationToken);
+	}
+}

app/Server/Endpoints/TrackChannelEndpoint.cs

@@ -1,5 +1,6 @@
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database;
@@ -9,7 +10,7 @@ using Microsoft.AspNetCore.Http;
 namespace DHT.Server.Endpoints;
 
 sealed class TrackChannelEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		var root = await ReadJson(request);
 		var server = ReadServer(root.RequireObject("server"), "server");
 		var channel = ReadChannel(root.RequireObject("channel"), "channel", server.Id);

app/Server/Endpoints/TrackMessagesEndpoint.cs

@@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Data.Filters;
@@ -19,7 +20,7 @@ sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
 	private const string HasNewMessages = "1";
 	private const string NoNewMessages = "0";
 
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		var root = await ReadJson(request);
 
 		if (root.ValueKind != JsonValueKind.Array) {
@@ -37,11 +38,11 @@ sealed class TrackMessagesEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
 		}
 
 		var addedMessageFilter = new MessageFilter { MessageIds = addedMessageIds };
-		bool anyNewMessages = await Db.Messages.Count(addedMessageFilter) < addedMessageIds.Count;
+		bool anyNewMessages = await Db.Messages.Count(addedMessageFilter, CancellationToken.None) < addedMessageIds.Count;
 
 		await Db.Messages.Add(messages);
 
-		await response.WriteTextAsync(anyNewMessages ? HasNewMessages : NoNewMessages);
+		await response.WriteTextAsync(anyNewMessages ? HasNewMessages : NoNewMessages, cancellationToken);
 	}
 
 	private static Message ReadMessage(JsonElement json, string path) => new () {

app/Server/Endpoints/TrackUsersEndpoint.cs

@@ -1,5 +1,6 @@
 using System.Net;
 using System.Text.Json;
+using System.Threading;
 using System.Threading.Tasks;
 using DHT.Server.Data;
 using DHT.Server.Database;
@@ -9,7 +10,7 @@ using Microsoft.AspNetCore.Http;
 namespace DHT.Server.Endpoints;
 
 sealed class TrackUsersEndpoint(IDatabaseFile db) : BaseEndpoint(db) {
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		var root = await ReadJson(request);
 
 		if (root.ValueKind != JsonValueKind.Array) {

app/Server/Endpoints/ViewerEndpoint.cs

@@ -16,13 +16,13 @@ sealed class ViewerEndpoint(IDatabaseFile db, ResourceLoader resources) : BaseEn
 	private readonly Dictionary<string, byte[]?> cache = new ();
 	private readonly SemaphoreSlim cacheSemaphore = new (1);
 
-	protected override async Task Respond(HttpRequest request, HttpResponse response) {
+	protected override async Task Respond(HttpRequest request, HttpResponse response, CancellationToken cancellationToken) {
 		string path = (string?) request.RouteValues["path"] ?? "index.html";
 		string resourcePath = "Viewer/" + path;
 		
 		byte[]? resourceBytes;
 
-		await cacheSemaphore.WaitAsync();
+		await cacheSemaphore.WaitAsync(cancellationToken);
 		try {
 			if (!cache.TryGetValue(resourcePath, out resourceBytes)) {
 				cache[resourcePath] = resourceBytes = await resources.ReadBytesAsyncIfExists(resourcePath);
@@ -36,7 +36,7 @@ sealed class ViewerEndpoint(IDatabaseFile db, ResourceLoader resources) : BaseEn
 		}
 		else {
 			var contentType = ContentTypeProvider.TryGetContentType(path, out string? type) ? type : null;
-			await response.WriteFileAsync(contentType, resourceBytes);
+			await response.WriteFileAsync(contentType, resourceBytes, cancellationToken);
 		}
 	}
 }

app/Server/Service/Middlewares/ServerLoggingMiddleware.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Diagnostics;
 using System.Threading.Tasks;
 using DHT.Utils.Logging;
@@ -6,24 +7,34 @@ using Microsoft.AspNetCore.Http.Extensions;
 
 namespace DHT.Server.Service.Middlewares;
 
-sealed class ServerLoggingMiddleware {
+sealed class ServerLoggingMiddleware(RequestDelegate next) {
 	private static readonly Log Log = Log.ForType<ServerLoggingMiddleware>();
-	
-	private readonly RequestDelegate next;
-	
-	public ServerLoggingMiddleware(RequestDelegate next) {
-		this.next = next;
-	}
 
 	public async Task InvokeAsync(HttpContext context) {
 		var stopwatch = Stopwatch.StartNew();
-		await next(context);
+		try {
+			await next(context);
+		} catch (OperationCanceledException) {
+			OnFinished(stopwatch, context);
+			throw;
+		}
+
+		OnFinished(stopwatch, context);
+	}
+
+	private static void OnFinished(Stopwatch stopwatch, HttpContext context) {
 		stopwatch.Stop();
-		
+
 		var request = context.Request;
 		var requestLength = request.ContentLength ?? 0L;
-		var responseStatus = context.Response.StatusCode;
 		var elapsedMs = stopwatch.ElapsedMilliseconds;
-		Log.Debug("Request to " + request.GetEncodedPathAndQuery() + " (" + requestLength + " B) returned " + responseStatus + ", took " + elapsedMs + " ms");
+
+		if (context.RequestAborted.IsCancellationRequested) {
+			Log.Debug("Request to " + request.GetEncodedPathAndQuery() + " (" + requestLength + " B) was cancelled after " + elapsedMs + " ms");
+		}
+		else {
+			var responseStatus = context.Response.StatusCode;
+			Log.Debug("Request to " + request.GetEncodedPathAndQuery() + " (" + requestLength + " B) returned " + responseStatus + ", took " + elapsedMs + " ms");
+		}
 	}
 }

app/Server/Service/ServerStartup.cs

@@ -51,7 +51,8 @@ sealed class Startup {
 		app.UseRouting();
 		app.UseEndpoints(endpoints => {
 			endpoints.MapGet("/get-tracking-script", new GetTrackingScriptEndpoint(db, parameters, resources).Handle);
-			endpoints.MapGet("/get-viewer-data", new GetViewerDataEndpoint(db, viewerSessions).Handle);
+			endpoints.MapGet("/get-viewer-metadata", new GetViewerMetadataEndpoint(db, viewerSessions).Handle);
+			endpoints.MapGet("/get-viewer-messages", new GetViewerMessagesEndpoint(db, viewerSessions).Handle);
 			endpoints.MapGet("/get-downloaded-file/{url}", new GetDownloadedFileEndpoint(db).Handle);
 			endpoints.MapPost("/track-channel", new TrackChannelEndpoint(db).Handle);
 			endpoints.MapPost("/track-users", new TrackUsersEndpoint(db).Handle);

app/Utils/Http/HttpExtensions.cs

@@ -1,33 +1,34 @@
 using System.IO;
 using System.Net.Mime;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 
 namespace DHT.Utils.Http; 
 
 public static class HttpExtensions {
-	public static Task WriteTextAsync(this HttpResponse response, string text) {
+	public static Task WriteTextAsync(this HttpResponse response, string text, CancellationToken cancellationToken) {
-		return WriteTextAsync(response, MediaTypeNames.Text.Plain, text);
+		return WriteTextAsync(response, MediaTypeNames.Text.Plain, text, cancellationToken);
 	}
 	
-	public static async Task WriteTextAsync(this HttpResponse response, string contentType, string text) {
+	public static async Task WriteTextAsync(this HttpResponse response, string contentType, string text, CancellationToken cancellationToken) {
 		response.ContentType = contentType;
-		await response.StartAsync();
+		await response.StartAsync(cancellationToken);
-		await response.WriteAsync(text, Encoding.UTF8);
+		await response.WriteAsync(text, Encoding.UTF8, cancellationToken);
 	}
 	
-	public static async Task WriteFileAsync(this HttpResponse response, string? contentType, byte[] bytes) {
+	public static async Task WriteFileAsync(this HttpResponse response, string? contentType, byte[] bytes, CancellationToken cancellationToken) {
 		response.ContentType = contentType ?? string.Empty;
 		response.ContentLength = bytes.Length;
-		await response.StartAsync();
+		await response.StartAsync(cancellationToken);
-		await response.Body.WriteAsync(bytes);
+		await response.Body.WriteAsync(bytes, cancellationToken);
 	}
 	
-	public static async Task WriteStreamAsync(this HttpResponse response, string? contentType, ulong? contentLength, Stream source) {
+	public static async Task WriteStreamAsync(this HttpResponse response, string? contentType, ulong? contentLength, Stream source, CancellationToken cancellationToken) {
 		response.ContentType = contentType ?? string.Empty;
 		response.ContentLength = (long?) contentLength;
-		await response.StartAsync();
+		await response.StartAsync(cancellationToken);
-		await source.CopyToAsync(response.Body);
+		await source.CopyToAsync(response.Body, cancellationToken);
 	}
 }