chylex/Brotli-Builder
1
0
Fork 0
mirror of https://github.com/chylex/Brotli-Builder.git synced 2025-05-13 08:34:05 +02:00
Code

Work on encoder/transformer API

Browse Source
This commit is contained in:
chylex 2019-11-27 00:30:25 +01:00
parent c9ad856c25
commit e8887b012a
27 changed files with 278 additions and 159 deletions

6
BrotliBuilder/State/BrotliFileController.cs
View File

@ -13,6 +13,8 @@ using BrotliLib.Serialization;
namespace BrotliBuilder.State{
sealed class BrotliFileController{
public BrotliSerializationParameters SerializationParameters { get; } = BrotliSerializationParameters.Default;
public BrotliCompressionParameters CompressionParameters { get; } = BrotliCompressionParameters.Default;
public MarkerLevel BitMarkerLevel { get; set; } = MarkerLevel.Verbose;
public event EventHandler<StateChangedEventArgs>? StateChanged;
@ -198,7 +200,7 @@ namespace BrotliBuilder.State{
private bool TryEncode(int token, byte[] bytes, BrotliFileParameters parameters, IBrotliEncoder encoder, out BrotliFileStructure file, out Stopwatch stopwatch){
try{
stopwatch = Stopwatch.StartNew();
file = BrotliFileStructure.FromEncoder(parameters, encoder, bytes);
file = BrotliFileStructure.FromEncoder(parameters, CompressionParameters, bytes, encoder);
stopwatch.Stop();
return true;
}catch(Exception ex){
@ -211,7 +213,7 @@ namespace BrotliBuilder.State{
private bool TryTransform(int token, BrotliFileStructure structure, IBrotliTransformer transformer, out BrotliFileStructure transformed, out Stopwatch stopwatch){
try{
stopwatch = Stopwatch.StartNew();
transformed = structure.Transform(transformer);
transformed = structure.Transform(transformer, CompressionParameters);
stopwatch.Stop();
return true;
}catch(Exception ex){

2
BrotliCalc/Commands/CmdBenchReserializeRebuild.cs
View File

@ -56,7 +56,7 @@ namespace BrotliCalc.Commands{
Stopwatch swRebuild = Stopwatch.StartNew();
try{
rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild()), Parameters.Serialization);
rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild(), Parameters.Compression), Parameters.Serialization);
}catch(Exception e){
Debug.WriteLine(e.ToString());
++failedFiles;

2
BrotliCalc/Commands/CmdTestEncoder.cs
View File

@ -35,7 +35,7 @@ namespace BrotliCalc.Commands{
protected override IEnumerable<object?[]> GenerateRows(BrotliFileGroup group, BrotliFile.Uncompressed file){
int? uncompressedBytes = file.SizeBytes;
int encodeBytes = group.CountBytesAndValidate(BrotliFileStructure.FromEncoder(Parameters.File, file.Contents, encoder!), Parameters.Serialization);
int encodeBytes = group.CountBytesAndValidate(BrotliFileStructure.FromEncoder(Parameters.File, Parameters.Compression, file.Contents, encoder!), Parameters.Serialization);
return new List<object?[]>{
new object?[]{ file.Name, uncompressedBytes, encodeBytes, encodeBytes - uncompressedBytes } // subtraction propagates null

2
BrotliCalc/Commands/CmdTestReserializeRebuild.cs
View File

@ -18,7 +18,7 @@ namespace BrotliCalc.Commands{
int? originalBytes = file.SizeBytes;
var reserializeBytes = group.CountBytesAndValidate(bfs, Parameters.Serialization);
var rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild()), Parameters.Serialization);
var rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild(), Parameters.Compression), Parameters.Serialization);
return new List<object?[]>{
new object?[]{ file.Name, file.Identifier, originalBytes, reserializeBytes, rebuildBytes, reserializeBytes - originalBytes, rebuildBytes - originalBytes } // subtraction propagates null

6
BrotliCalc/Commands/CmdTestTransformer.cs
View File

@ -33,14 +33,14 @@ namespace BrotliCalc.Commands{
protected override IEnumerable<object?[]> GenerateRows(BrotliFileGroup group, BrotliFile.Compressed file){
var bfs = file.Structure;
var transformed = bfs.Transform(transformer!);
var transformed = bfs.Transform(transformer!, Parameters.Compression);
if (transformed.MetaBlocks.SequenceEqual(bfs.MetaBlocks)){ // if the references have not changed, there was no transformation
return new List<object[]>();
}
int? originalBytes = file.SizeBytes;
int rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild()), Parameters.Serialization);
int rebuildBytes = group.CountBytesAndValidate(bfs.Transform(new TransformRebuild(), Parameters.Compression), Parameters.Serialization);
int transformedBytes = group.CountBytesAndValidate(transformed, Parameters.Serialization);
return new List<object?[]>{

1
BrotliCalc/Parameters.cs
View File

@ -4,5 +4,6 @@ namespace BrotliCalc{
static class Parameters{
public static BrotliFileParameters File { get; } = BrotliFileParameters.Default;
public static BrotliSerializationParameters Serialization { get; } = BrotliSerializationParameters.Default;
public static BrotliCompressionParameters Compression { get; } = BrotliCompressionParameters.Default;
}
}

30
BrotliImpl/Encoders/EncodeGreedySearch.cs
View File

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using BrotliImpl.Encoders.Utils;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Compressed;
using BrotliLib.Brotli.Components.Data;
@ -14,7 +13,7 @@ namespace BrotliImpl.Encoders{
/// Encodes bytes into a series of compressed meta-blocks. For each byte, it attempts to find the nearest and longest copy within the sliding window, or dictionary word.
/// </summary>
public abstract class EncodeGreedySearch : IBrotliEncoder{
private protected abstract Copy? FindCopy(BrotliFileParameters parameters, byte[] bytes, int start, int maxLength);
private protected abstract Copy? FindCopy(BrotliFileParameters parameters, ArraySegment<byte> bytes, int start, int maxLength);
// Implementations
@ -25,8 +24,8 @@ namespace BrotliImpl.Encoders{
this.minLength = Math.Max(minLength, InsertCopyLengths.MinCopyLength);
}
private protected override Copy? FindCopy(BrotliFileParameters parameters, byte[] bytes, int start, int maxLength){
int length = bytes.Length;
private protected override Copy? FindCopy(BrotliFileParameters parameters, ArraySegment<byte> bytes, int start, int maxLength){
int length = bytes.Count;
if (start < InsertCopyLengths.MinCopyLength || start >= length - InsertCopyLengths.MinCopyLength || maxLength < InsertCopyLengths.MinCopyLength){
return null;
@ -52,8 +51,8 @@ namespace BrotliImpl.Encoders{
}
public sealed class OnlyDictionary : EncodeGreedySearch{
private protected override Copy? FindCopy(BrotliFileParameters parameters, byte[] bytes, int start, int maxLength){
var entries = parameters.Dictionary.Index.Find(bytes, start, maxLength);
private protected override Copy? FindCopy(BrotliFileParameters parameters, ArraySegment<byte> bytes, int start, int maxLength){
var entries = parameters.Dictionary.Index.Find(bytes.Slice(start), maxLength);
if (entries.Count == 0){
return null;
@ -72,7 +71,7 @@ namespace BrotliImpl.Encoders{
this.findDictionary = new OnlyDictionary();
}
private protected override Copy? FindCopy(BrotliFileParameters parameters, byte[] bytes, int start, int maxLength){
private protected override Copy? FindCopy(BrotliFileParameters parameters, ArraySegment<byte> bytes, int start, int maxLength){
Copy? found1 = findBackReferences.FindCopy(parameters, bytes, start, maxLength);
Copy? found2 = findDictionary.FindCopy(parameters, bytes, start, maxLength);
@ -82,14 +81,15 @@ namespace BrotliImpl.Encoders{
// Generation
public IEnumerable<MetaBlock> GenerateMetaBlocks(BrotliFileParameters parameters, byte[] bytes){
var builder = new CompressedMetaBlockBuilder(parameters);
int length = bytes.Length;
public (MetaBlock, BrotliEncodeInfo) Encode(BrotliEncodeInfo info){
var bytes = info.Bytes;
int length = bytes.Count;
var builder = info.NewBuilder();
var nextLiteralBatch = new List<Literal>();
for(int index = 0; index < length;){
var copy = FindCopy(parameters, bytes, index, DataLength.MaxUncompressedBytes - nextLiteralBatch.Count);
var copy = FindCopy(info.FileParameters, bytes, index, DataLength.MaxUncompressedBytes - nextLiteralBatch.Count);
int mbSize;
if (copy == null){
@ -99,15 +99,13 @@ namespace BrotliImpl.Encoders{
mbSize = nextLiteralBatch.Count;
}
else{
index += copy.AddCommand(parameters, builder, nextLiteralBatch);
index += copy.AddCommand(info.FileParameters, builder, nextLiteralBatch);
nextLiteralBatch.Clear();
mbSize = builder.OutputSize;
}
if (mbSize == DataLength.MaxUncompressedBytes){
var (mb, next) = builder.Build();
builder = next();
yield return mb;
return builder.Build(info);
}
}
@ -115,7 +113,7 @@ namespace BrotliImpl.Encoders{
builder.AddInsertCopy(new InsertCopyCommand(nextLiteralBatch));
}
yield return builder.Build().MetaBlock;
return builder.Build(info);
}
}
}

33
BrotliImpl/Encoders/EncodeLiterals.cs
View File

@ -1,12 +1,8 @@
using System;
using System.Collections.Generic;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Compressed;
using BrotliLib.Brotli.Components.Data;
using BrotliLib.Brotli.Components.Header;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Collections;
namespace BrotliImpl.Encoders{
@ -14,29 +10,12 @@ namespace BrotliImpl.Encoders{
/// Encodes bytes into a series of compressed meta-blocks, where each contains a single insert&amp;copy command with each byte stored as a literal.
/// </summary>
public class EncodeLiterals : IBrotliEncoder{
public IEnumerable<MetaBlock> GenerateMetaBlocks(BrotliFileParameters parameters, byte[] bytes){
int length = bytes.Length;
var builder = new CompressedMetaBlockBuilder(parameters);
public (MetaBlock, BrotliEncodeInfo) Encode(BrotliEncodeInfo info){
var bytes = CollectionHelper.SliceAtMost(info.Bytes, DataLength.MaxUncompressedBytes).ToArray();
for(int index = 0, nextIndex; index < length; index = nextIndex){
nextIndex = index + DataLength.MaxUncompressedBytes;
int mbBytes = Math.Min(length - index, DataLength.MaxUncompressedBytes);
byte[] mbData = CollectionHelper.Slice(bytes, index, mbBytes);
var (mb, next) = builder.AddInsertCopy(new InsertCopyCommand(Literal.FromBytes(mbData), InsertCopyLengths.MinCopyLength))
.Build();
if (nextIndex < length){
builder = next();
}
yield return mb;
}
if (length == 0){
yield return new MetaBlock.LastEmpty();
}
return info.NewBuilder()
.AddInsertCopy(new InsertCopyCommand(Literal.FromBytes(bytes), InsertCopyLengths.MinCopyLength))
.Build(info);
}
}
}

21
BrotliImpl/Encoders/EncodeUncompressedOnly.cs
View File

@ -1,26 +1,19 @@
using System;
using System.Collections.Generic;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Header;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Collections;
namespace BrotliImpl.Encoders{
/// <summary>
/// Encodes bytes into a series of uncompressed meta-blocks, with an empty meta-block at the end.
/// Encodes bytes into a series of uncompressed meta-blocks.
/// </summary>
public class EncodeUncompressedOnly : IBrotliEncoder{
public IEnumerable<MetaBlock> GenerateMetaBlocks(BrotliFileParameters parameters, byte[] bytes){
for(int index = 0; index < bytes.Length; index += DataLength.MaxUncompressedBytes){
int mbBytes = Math.Min(bytes.Length - index, DataLength.MaxUncompressedBytes);
byte[] mbData = CollectionHelper.Slice(bytes, index, mbBytes);
public (MetaBlock, BrotliEncodeInfo) Encode(BrotliEncodeInfo info){
var state = info.State;
var bytes = CollectionHelper.SliceAtMost(info.Bytes, DataLength.MaxUncompressedBytes).ToArray();
yield return new MetaBlock.Uncompressed(mbData);
}
yield return new MetaBlock.LastEmpty();
state.OutputBytes(bytes);
return (new MetaBlock.Uncompressed(bytes), info.WithProcessedBytes(state, bytes.Length));
}
}
}

2
BrotliImpl/Encoders/Utils/Copy.cs
View File

@ -2,7 +2,7 @@
using BrotliLib.Brotli.Components.Compressed;
using BrotliLib.Brotli.Components.Data;
using BrotliLib.Brotli.Dictionary.Index;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Parameters;
namespace BrotliImpl.Encoders.Utils{

19
BrotliImpl/MetaBlockSizeTracker.cs
View File

@ -2,13 +2,13 @@
using System.Diagnostics;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Serialization.Writer;
namespace BrotliImpl{
class MetaBlockSizeTracker{
public MetaBlock? Smallest { get; private set; } = null;
public (MetaBlock, BrotliGlobalState)? Smallest { get; private set; } = null;
public int SmallestSize { get; private set; } = int.MaxValue;
private readonly BrotliGlobalState initialState;
@ -18,7 +18,7 @@ namespace BrotliImpl{
}
public void Test(MetaBlock tested, BrotliSerializationParameters? serializationParameters = null, string? debugText = null){
int testedSize = CountBits(tested, initialState.Clone(), serializationParameters);
var (testedSize, nextState) = CountBits(tested, initialState.Clone(), serializationParameters) ?? (int.MaxValue, null!);
if (debugText != null){
Debug.Write(debugText + " = " + testedSize + " bits");
@ -29,7 +29,7 @@ namespace BrotliImpl{
Debug.Write(" < " + SmallestSize + " bits (new best)");
}
Smallest = tested;
Smallest = (tested, nextState);
SmallestSize = testedSize;
}
@ -38,19 +38,20 @@ namespace BrotliImpl{
}
}
public void Test(CompressedMetaBlockBuilder builder, BrotliSerializationParameters? serializationParameters = null, string? debugText = null){
Test(builder.Build().MetaBlock, serializationParameters, debugText);
public void Test(CompressedMetaBlockBuilder builder, BrotliCompressionParameters compressionParameters, BrotliSerializationParameters? serializationParameters = null, string? debugText = null){
Test(builder.Build(compressionParameters).MetaBlock, serializationParameters, debugText);
}
public static int CountBits(MetaBlock tested, BrotliGlobalState state, BrotliSerializationParameters? serializationParameters = null){
public static (int, BrotliGlobalState)? CountBits(MetaBlock tested, BrotliGlobalState state, BrotliSerializationParameters? serializationParameters = null){
var writer = new BitWriterNull();
var nextState = state.Clone();
try{
return writer.Length;
MetaBlock.Serialize(writer, tested, nextState, serializationParameters ?? BrotliSerializationParameters.Default);
return (writer.Length, nextState);
}catch(Exception ex){
Debug.WriteLine(ex.ToString());
return int.MaxValue;
return null;
}
}
}

11
BrotliImpl/Transformers/TransformRebuild.cs
View File

@ -1,12 +1,13 @@
using System.Collections.Generic;
using BrotliLib.Brotli;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Parameters;
namespace BrotliImpl.Transformers{
public class TransformRebuild : CompressedMetaBlockTransformer{
protected override IEnumerable<MetaBlock> Transform(MetaBlock.Compressed original, CompressedMetaBlockBuilder builder, BrotliGlobalState initialState){
yield return builder.Build().MetaBlock;
public class TransformRebuild : BrotliTransformerCompressed{
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return new CompressedMetaBlockBuilder(original, state).Build(parameters);
}
}
}

18
BrotliImpl/Transformers/TransformSplitInsertCopyLengths.cs
View File

@ -1,33 +1,33 @@
using System.Collections.Generic;
using System.Linq;
using System.Linq;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Compressed;
using BrotliLib.Brotli.Components.Utils;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Parameters;
namespace BrotliImpl.Transformers{
public class TransformSplitInsertCopyLengths : CompressedMetaBlockTransformer{
protected override IEnumerable<MetaBlock> Transform(MetaBlock.Compressed original, CompressedMetaBlockBuilder builder, BrotliGlobalState initialState){
public class TransformSplitInsertCopyLengths : BrotliTransformerCompressed{
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
var builder = new CompressedMetaBlockBuilder(original, state);
var blockTypes = builder.BlockTypes[Category.InsertCopy];
if (blockTypes.Commands.Any()){
yield return original;
yield break;
return (original, state);
}
var icCommands = builder.InsertCopyCommands.Count();
var icSwitchAt = icCommands / 2;
if (icCommands <= 1){
yield return original;
yield break;
return (original, state);
}
blockTypes.SetInitialLength(icSwitchAt)
.AddBlockSwitch(new BlockSwitchCommand(1, icCommands - icSwitchAt));
yield return builder.Build().MetaBlock;
return builder.Build(parameters);
}
}
}

15
BrotliImpl/Transformers/TransformTestDistanceParameters.cs
View File

@ -1,22 +1,25 @@
using System.Collections.Generic;
using System;
using BrotliLib.Brotli;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Header;
using BrotliLib.Brotli.Encode;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Parameters;
namespace BrotliImpl.Transformers{
public class TransformTestDistanceParameters : CompressedMetaBlockTransformer{
protected override IEnumerable<MetaBlock> Transform(MetaBlock.Compressed original, CompressedMetaBlockBuilder builder, BrotliGlobalState initialState){
var tracker = new MetaBlockSizeTracker(initialState);
public class TransformTestDistanceParameters : BrotliTransformerCompressed{
protected override (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
var builder = new CompressedMetaBlockBuilder(original, state);
var tracker = new MetaBlockSizeTracker(state);
for(byte postfixBitCount = 0; postfixBitCount <= DistanceParameters.MaxPostfixBitCount; postfixBitCount++){
for(byte directCodeBits = 0; directCodeBits <= DistanceParameters.MaxDirectCodeBits; directCodeBits++){
builder.DistanceParameters = new DistanceParameters(postfixBitCount, directCodeBits);
tracker.Test(builder, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]");
tracker.Test(builder, parameters, debugText: "[PostfixBitCount = " + postfixBitCount + ", DirectCodeBits = " + directCodeBits + "]");
}
}
yield return tracker.Smallest!;
return tracker.Smallest ?? throw new InvalidOperationException("Transformation did not generate any meta-blocks.");
}
}
}

29
BrotliLib/Brotli/BrotliFileStructure.cs
View File

@ -9,7 +9,6 @@ using BrotliLib.Brotli.Parameters;
using BrotliLib.Markers;
using BrotliLib.Markers.Serialization.Reader;
using BrotliLib.Serialization;
using BrotliLib.Serialization.Writer;
namespace BrotliLib.Brotli{
/// <summary>
@ -27,37 +26,29 @@ namespace BrotliLib.Brotli{
return DoDeserialize(CreateReader(new BitStream(bytes)), new FileContext(BrotliDefaultDictionary.Embedded, windowSize => new BrotliOutputWindowed(windowSize)));
}
public static BrotliFileStructure FromEncoder(BrotliFileParameters parameters, IBrotliEncoder encoder, byte[] bytes){
var bfs = new BrotliFileStructure(parameters);
foreach(MetaBlock metaBlock in encoder.GenerateMetaBlocks(parameters, bytes)){
bfs.MetaBlocks.Add(metaBlock);
}
bfs.Fixup();
return bfs;
public static BrotliFileStructure FromEncoder(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, byte[] bytes, IBrotliEncoder encoder, params IBrotliTransformer[] transformers){
return new BrotliEncodePipeline(encoder, transformers).Apply(fileParameters, compressionParameters, bytes);
}
// Data
public BrotliFileParameters Parameters { get; set; }
public IList<MetaBlock> MetaBlocks { get; }
public List<MetaBlock> MetaBlocks { get; }
public BrotliFileStructure(BrotliFileParameters parameters){
this.Parameters = parameters;
this.MetaBlocks = new List<MetaBlock>();
}
public BrotliFileStructure Transform(IBrotliTransformer transformer, BrotliSerializationParameters? parameters = null){
public BrotliFileStructure Transform(IBrotliTransformer transformer, BrotliCompressionParameters compressionParameters){
var copy = new BrotliFileStructure(Parameters);
var state = new BrotliGlobalState(Parameters, new BrotliOutputWindowed(Parameters.WindowSize));
var writer = new BitWriterNull();
foreach(MetaBlock original in MetaBlocks){
foreach(MetaBlock transformed in transformer.Transform(original, state)){ // TODO figure out how to handle state
copy.MetaBlocks.Add(transformed);
MetaBlock.Serialize(writer, transformed, state, parameters ?? BrotliSerializationParameters.Default);
}
var (transformedMetaBlocks, transformedState) = transformer.Transform(original, state, compressionParameters);
copy.MetaBlocks.AddRange(transformedMetaBlocks);
state = transformedState;
}
copy.Fixup();
@ -65,6 +56,10 @@ namespace BrotliLib.Brotli{
}
public void Fixup(){
if (MetaBlocks.Count == 0 || MetaBlocks[^1] is MetaBlock.Uncompressed){
MetaBlocks.Add(new MetaBlock.LastEmpty());
}
for(int index = 0, last = MetaBlocks.Count - 1; index <= last; index++){
MetaBlocks[index].IsLast = index == last;
}

22
BrotliLib/Brotli/Dictionary/Index/BrotliDictionaryIndex.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using BrotliLib.Brotli.Dictionary.Format;
@ -45,14 +46,14 @@ namespace BrotliLib.Brotli.Dictionary.Index{
Debug.WriteLine("Constructed dictionary index in " + sw.ElapsedMilliseconds + " ms.");
}
public List<DictionaryIndexEntry> Find(byte[] bytes, int start, int maxLength){
public List<DictionaryIndexEntry> Find(ArraySegment<byte> bytes, int maxLength = int.MaxValue){
var entries = new List<DictionaryIndexEntry>();
// default dictionary guarantees executing 44 identity, 12 ferment first, 12 ferment all transformations
// TODO longest may not find correct entries w/ suffix, but it seems to work well enough
var identityNoPrefixWords = lookups[TransformType.Identity].FindLongest(CollectionHelper.Skip(bytes, start));
var fermentFirstNoPrefixWords = lookups[TransformType.FermentFirst].FindLongest(CollectionHelper.Skip(bytes, start));
var fermentAllNoPrefixWords = lookups[TransformType.FermentAll].FindLongest(CollectionHelper.Skip(bytes, start));
var identityNoPrefixWords = lookups[TransformType.Identity].FindLongest(bytes);
var fermentFirstNoPrefixWords = lookups[TransformType.FermentFirst].FindLongest(bytes);
var fermentAllNoPrefixWords = lookups[TransformType.FermentAll].FindLongest(bytes);
IEnumerable<(int, int)> LookupWords(TransformType type, int prefixLength){
if (prefixLength == 0){
@ -63,10 +64,15 @@ namespace BrotliLib.Brotli.Dictionary.Index{
}
}
return lookups[type].FindLongest(CollectionHelper.Skip(bytes, start + prefixLength));
if (prefixLength <= bytes.Count){
return lookups[type].FindLongest(bytes.Slice(prefixLength));
}
else{
return Enumerable.Empty<(int, int)>();
}
}
var transformsMatchingPrefix = transformsNoPrefix.Concat(transformsWithPrefix.Where(index => CollectionHelper.ContainsAt(bytes, start, transforms[index].Prefix)));
var transformsMatchingPrefix = transformsNoPrefix.Concat(transformsWithPrefix.Where(index => CollectionHelper.ContainsAt(bytes, 0, transforms[index].Prefix)));
foreach(var transform in transformsMatchingPrefix){
var wt = transforms[transform];
@ -77,7 +83,7 @@ namespace BrotliLib.Brotli.Dictionary.Index{
foreach(var (length, word) in LookupWords(type, prefixLength)){
var transformedLength = type.GetTransformedLength(length);
if (transformedLength <= maxLength && CollectionHelper.ContainsAt(bytes, start + prefixLength + transformedLength, wt.Suffix)){
if (transformedLength <= maxLength && CollectionHelper.ContainsAt(bytes, prefixLength + transformedLength, wt.Suffix)){
int packedValue = format.GetPackedValue(length, word, transform);
int outputLength = transformedLength + prefixLength + wt.Suffix.Length;

45
BrotliLib/Brotli/Encode/BrotliEncodeInfo.cs Normal file
View File

@ -0,0 +1,45 @@
using System;
using BrotliLib.Brotli.Encode.Build;
using BrotliLib.Brotli.Output;
using BrotliLib.Brotli.Parameters;
namespace BrotliLib.Brotli.Encode{
public sealed class BrotliEncodeInfo{
public BrotliFileParameters FileParameters { get; }
public BrotliCompressionParameters CompressionParameters { get; }
public BrotliGlobalState State => state.Clone();
public ArraySegment<byte> Bytes { get; }
public bool IsFinished => Bytes.Count == 0;
private readonly BrotliGlobalState state;
private BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, BrotliGlobalState state, ArraySegment<byte> bytes){
this.FileParameters = fileParameters;
this.CompressionParameters = compressionParameters;
this.state = state;
this.Bytes = bytes;
}
public BrotliEncodeInfo(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, byte[] bytes) : this(
fileParameters,
compressionParameters,
new BrotliGlobalState(fileParameters, new BrotliOutputWindowed(fileParameters.WindowSize)),
new ArraySegment<byte>(bytes)
){}
public CompressedMetaBlockBuilder NewBuilder(){
return new CompressedMetaBlockBuilder(state);
}
public BrotliEncodeInfo WithState(BrotliGlobalState newState){
return new BrotliEncodeInfo(FileParameters, CompressionParameters, newState, Bytes);
}
public BrotliEncodeInfo WithProcessedBytes(BrotliGlobalState newState, int processedBytes){
return new BrotliEncodeInfo(FileParameters, CompressionParameters, newState, Bytes.Slice(processedBytes));
}
}
}

43
BrotliLib/Brotli/Encode/BrotliEncodePipeline.cs Normal file
View File

@ -0,0 +1,43 @@
using System.Linq;
using BrotliLib.Brotli.Parameters;
namespace BrotliLib.Brotli.Encode{
public sealed class BrotliEncodePipeline{
private readonly IBrotliEncoder encoder;
private readonly IBrotliTransformer[] transformers;
public BrotliEncodePipeline(IBrotliEncoder encoder, params IBrotliTransformer[] transformers){
this.encoder = encoder;
this.transformers = transformers.ToArray();
}
public BrotliFileStructure Apply(BrotliFileParameters fileParameters, BrotliCompressionParameters compressionParameters, byte[] bytes){
var bfs = new BrotliFileStructure(fileParameters);
var encodeInfo = new BrotliEncodeInfo(fileParameters, compressionParameters, bytes);
do{
var (metaBlock, newEncodeInfo) = encoder.Encode(encodeInfo);
if (transformers.Length == 0){
bfs.MetaBlocks.Add(metaBlock);
encodeInfo = newEncodeInfo;
}
else{
var state = newEncodeInfo.State;
foreach(var transformer in transformers){
var (transformedMetaBlocks, transformedState) = transformer.Transform(metaBlock, state, compressionParameters);
bfs.MetaBlocks.AddRange(transformedMetaBlocks);
state = transformedState;
}
encodeInfo = encodeInfo.WithState(state);
}
}while(!encodeInfo.IsFinished);
bfs.Fixup();
return bfs;
}
}
}

38
BrotliLib/Brotli/Encode/BrotliTransformerBase.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Components.Compressed;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Serialization.Writer;
namespace BrotliLib.Brotli.Encode{
public abstract class BrotliTransformerBase : IBrotliTransformer{
(IList<MetaBlock> MetaBlocks, BrotliGlobalState NextState) IBrotliTransformer.Transform(MetaBlock original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return original switch{
MetaBlock.LastEmpty le => TransformLastEmpty(le, state, parameters),
MetaBlock.PaddedEmpty pe => TransformPaddedEmpty(pe, state, parameters),
MetaBlock.Uncompressed u => TransformUncompressed(u, state, parameters),
MetaBlock.Compressed c => TransformCompressed(c, state, parameters),
_ => throw new InvalidOperationException("Unknown meta-block type: " + original.GetType().Name)
};
}
protected virtual (IList<MetaBlock>, BrotliGlobalState) TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return (Array.Empty<MetaBlock>(), state);
}
protected virtual (IList<MetaBlock>, BrotliGlobalState) TransformPaddedEmpty(MetaBlock.PaddedEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return (new MetaBlock[]{ original }, state);
}
protected virtual (IList<MetaBlock>, BrotliGlobalState) TransformUncompressed(MetaBlock.Uncompressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
state.OutputBytes(original.UncompressedData);
return (new MetaBlock[]{ original }, state);
}
protected virtual (IList<MetaBlock>, BrotliGlobalState) TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
MetaBlockCompressionData.Serialize(new BitWriterNull(), original.Data, new MetaBlockCompressionData.Context(original.Header, original.DataLength, state));
return (new MetaBlock[]{ original }, state);
}
}
}

26
BrotliLib/Brotli/Encode/BrotliTransformerCompressed.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Collections.Generic;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Parameters;
namespace BrotliLib.Brotli.Encode{
public abstract class BrotliTransformerCompressed : BrotliTransformerBase{
protected sealed override (IList<MetaBlock>, BrotliGlobalState) TransformLastEmpty(MetaBlock.LastEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return base.TransformLastEmpty(original, state, parameters);
}
protected sealed override (IList<MetaBlock>, BrotliGlobalState) TransformPaddedEmpty(MetaBlock.PaddedEmpty original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return base.TransformPaddedEmpty(original, state, parameters);
}
protected sealed override (IList<MetaBlock>, BrotliGlobalState) TransformUncompressed(MetaBlock.Uncompressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
return base.TransformUncompressed(original, state, parameters);
}
protected sealed override (IList<MetaBlock>, BrotliGlobalState) TransformCompressed(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters){
var (metaBlock, nextState) = Transform(original, state, parameters);
return (new MetaBlock[]{ metaBlock }, nextState);
}
protected abstract (MetaBlock, BrotliGlobalState) Transform(MetaBlock.Compressed original, BrotliGlobalState state, BrotliCompressionParameters parameters);
}
}

2
BrotliLib/Brotli/Encode/BlockSwitchBuilder.cs → BrotliLib/Brotli/Encode/Build/BlockSwitchBuilder.cs
View File

@ -7,7 +7,7 @@ using BrotliLib.Brotli.Components.Header;
using BrotliLib.Brotli.Components.Utils;
using BrotliLib.Collections;
namespace BrotliLib.Brotli.Encode{
namespace BrotliLib.Brotli.Encode.Build{
public sealed class BlockSwitchBuilder{
public int InitialLength { get; private set; }
public IEnumerable<BlockSwitchCommand> Commands => commands;

17
BrotliLib/Brotli/Encode/CompressedMetaBlockBuilder.cs → BrotliLib/Brotli/Encode/Build/CompressedMetaBlockBuilder.cs
View File

@ -11,7 +11,7 @@ using BrotliLib.Brotli.Output;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Collections;
namespace BrotliLib.Brotli.Encode{
namespace BrotliLib.Brotli.Encode.Build{
public sealed class CompressedMetaBlockBuilder{
public int OutputSize => intermediateState.OutputSize - initialState.OutputSize;
@ -31,7 +31,7 @@ namespace BrotliLib.Brotli.Encode{
// Construction
private CompressedMetaBlockBuilder(BrotliGlobalState state){
public CompressedMetaBlockBuilder(BrotliGlobalState state){
this.initialState = state.Clone();
this.intermediateState = state.Clone();
}
@ -83,9 +83,14 @@ namespace BrotliLib.Brotli.Encode{
// Building
public (MetaBlock MetaBlock, Func<CompressedMetaBlockBuilder> Next) Build(){
var state = initialState.Clone();
public (MetaBlock MetaBlock, BrotliEncodeInfo NextInfo) Build(BrotliEncodeInfo info){
var (metaBlock, nextState) = Build(info.CompressionParameters);
return (metaBlock, info.WithProcessedBytes(nextState, OutputSize));
}
public (MetaBlock MetaBlock, BrotliGlobalState NextState) Build(BrotliCompressionParameters parameters){
var state = initialState.Clone();
// Command processing
var bsCommands = BlockTypes.Select<IList<BlockSwitchCommand>>(builder => new List<BlockSwitchCommand>(builder.Commands));
@ -129,7 +134,7 @@ namespace BrotliLib.Brotli.Encode{
int treeID = DistanceCtxMap.DetermineTreeID(blockID, contextID);
var codeList = distanceCodeFreq[treeID];
codeList.Add(distanceCode = distanceCodes.FirstOrDefault(codeList.Contains) ?? distanceCodes[0]); // TODO figure out a better strategy for picking the code
codeList.Add(distanceCode = distanceCodes.FirstOrDefault(codeList.Contains) ?? distanceCodes[0]); // new CompressedMetaBlockBuildParams().PickDistanceCode(icCommand.CopyDistance, distanceCodes, codeList));
}
bool isImplicitCodeZero = distanceCode == null;
@ -183,7 +188,7 @@ namespace BrotliLib.Brotli.Encode{
var data = new MetaBlockCompressionData(icCommandsFinal, bsCommands);
var dataLength = new DataLength(OutputSize);
return (new MetaBlock.Compressed(isLast: false, dataLength, header, data), () => new CompressedMetaBlockBuilder(state));
return (new MetaBlock.Compressed(isLast: false, dataLength, header, data), state);
}
// Helpers

19
BrotliLib/Brotli/Encode/CompressedMetaBlockTransformer.cs
View File

@ -1,19 +0,0 @@
using System.Collections.Generic;
using BrotliLib.Brotli.Components;
namespace BrotliLib.Brotli.Encode{
public abstract class CompressedMetaBlockTransformer : IBrotliTransformer{
public IEnumerable<MetaBlock> Transform(MetaBlock original, BrotliGlobalState initialState){
if (!(original is MetaBlock.Compressed compressed)){
yield return original;
yield break;
}
foreach(MetaBlock transformed in Transform(compressed, new CompressedMetaBlockBuilder(compressed, initialState), initialState)){
yield return transformed;
}
}
protected abstract IEnumerable<MetaBlock> Transform(MetaBlock.Compressed original, CompressedMetaBlockBuilder builder, BrotliGlobalState initialState);
}
}

6
BrotliLib/Brotli/Encode/IBrotliEncoder.cs
View File

@ -1,12 +1,10 @@
using System.Collections.Generic;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Parameters;
using BrotliLib.Brotli.Components;
namespace BrotliLib.Brotli.Encode{
/// <summary>
/// Allows converting bytes into a series of <see cref="MetaBlock"/> objects.
/// </summary>
public interface IBrotliEncoder{
IEnumerable<MetaBlock> GenerateMetaBlocks(BrotliFileParameters parameters, byte[] bytes);
(MetaBlock MetaBlock, BrotliEncodeInfo Next) Encode(BrotliEncodeInfo info);
}
}

3
BrotliLib/Brotli/Encode/IBrotliTransformer.cs
View File

@ -1,8 +1,9 @@
using System.Collections.Generic;
using BrotliLib.Brotli.Components;
using BrotliLib.Brotli.Parameters;
namespace BrotliLib.Brotli.Encode{
public interface IBrotliTransformer{
IEnumerable<MetaBlock> Transform(MetaBlock original, BrotliGlobalState initialState);
(IList<MetaBlock> MetaBlocks, BrotliGlobalState NextState) Transform(MetaBlock original, BrotliGlobalState state, BrotliCompressionParameters parameters);
}
}

5
BrotliLib/Brotli/Parameters/BrotliCompressionParameters.cs Normal file
View File

@ -0,0 +1,5 @@
namespace BrotliLib.Brotli.Parameters{
public sealed class BrotliCompressionParameters{
public static BrotliCompressionParameters Default => new BrotliCompressionParameters();
}
}

14
BrotliLib/Collections/CollectionHelper.cs
View File

@ -13,13 +13,17 @@ namespace BrotliLib.Collections{
public static byte[] Clone(byte[] input){
return Slice(input, 0, input.Length);
}
public static ArraySegment<byte> SliceAtMost(ArraySegment<byte> input, int count){
return input.Slice(0, Math.Min(input.Count, count));
}
public static bool ContainsAt(byte[] input, int start, byte[] contents){
public static bool ContainsAt(ArraySegment<byte> input, int start, byte[] contents){
if (contents.Length == 0){
return true;
}
if (input.Length - start < contents.Length){
if (input.Count - start < contents.Length){
return false;
}
@ -32,12 +36,6 @@ namespace BrotliLib.Collections{
return true;
}
public static IEnumerable<T> Skip<T>(T[] array, int start){
for(int index = start; index < array.Length; index++){
yield return array[index];
}
}
public static bool Equal(byte[] a, byte[] b){
return new ReadOnlySpan<byte>(a).SequenceEqual(b);
}