Skip to content

Commit bc4fcc0

Browse files
Nick-NuonNick-Nuon
authored
Merge pull request #29 from simdutf/Arm
64-bit ARM support
2 parents e59e906 + c0a09a1 commit bc4fcc0

15 files changed

+2274
-259
lines changed

‎.github/workflows/dotnet.yml

+3-2
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@ jobs:
1313
steps:
1414
- uses: actions/checkout@v4
1515

16-
- name: Setup .NET 8.0
16+
- name: Setup .NET 9.0 (preview)
1717
uses: actions/setup-dotnet@v4
1818
with:
19-
dotnet-version: '8.0.x'
19+
dotnet-version: 9.0.x
20+
dotnet-quality: preview
2021
- name: Restore dependencies
2122
run: dotnet restore
2223
- name: Build

‎README.md

+31-4
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,43 @@
22
## Fast WHATWG forgiving-base64 in C#
33

44
The C# standard library has fast (SIMD-based) base64 encoding functions, but it lacks
5-
base64 decoding function. The initial work that lead to the fast functions in the runtime
6-
was carried out by [gfoidl](https://github.com/gfoidl/Base64).
5+
really fast base64 decoding function. The initial work that lead to the fast functions in the runtime
6+
was carried out by [gfoidl](https://github.com/gfoidl/Base64).
7+
8+
- There are accelerated base64 functions for UTF-8 inputs in the .NET runtime, but they are not optimal:
9+
we can make them 50% to 2x or 3x faster.
10+
- There is no accelerated base64 functions for UTF-16 inputs (e.g., `string` types). We can be 2x faster
11+
or more.
712

813
The goal of this project is to provide the fast WHATWG forgiving-base64 algorithm already
9-
used in major JavaScript runtimes (Node.js and Bun) to C#. It would complete the existing work.
14+
used in major JavaScript runtimes (Node.js and Bun) to C#.
15+
16+
Importantly, we only focus on base64 decoding. It is a more challenging problem than base64 encoding because
17+
of the presence of allowable white space characters and the need to validate the input. Indeed, all
18+
inputs are valid for encoding, but only some inputs are valid for decoding. Having to skip white space
19+
characters makes accelerated decoding somewhat difficult.
1020

1121

1222
## Requirements
1323

14-
We recommend you install .NET 8 or better: https://dotnet.microsoft.com/en-us/download/dotnet/8.0
24+
We require .NET 9 or better: https://dotnet.microsoft.com/en-us/download/dotnet/9.0
25+
26+
27+
## Usage
28+
29+
The library only provides Base64 decoding functions, because the .NET library already has
30+
fast Base64 encoding functions.
31+
32+
```c#
33+
string base64 = "SGVsbG8sIFdvcmxkIQ==";
34+
byte[] buffer = new byte[SimdBase64.Base64.MaximalBinaryLengthFromBase64(base64.AsSpan())];
35+
int bytesConsumed; // gives you the number of characters consumed
36+
int bytesWritten;
37+
var result = SimdBase64.Base64.DecodeFromBase64(base64.AsSpan(), buffer, out bytesConsumed, out bytesWritten, false); // false is for regular base64, true for base64url
38+
// result == OperationStatus.Done
39+
// Encoding.UTF8.GetString(buffer.AsSpan().Slice(0, bytesWritten)) == "Hello, World!"
40+
41+
```
1542

1643

1744
## Running tests

‎benchmark/Benchmark.cs

+117-44
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,19 @@
1-
using System;
2-
using BenchmarkDotNet.Attributes;
1+
using BenchmarkDotNet.Attributes;
32
using BenchmarkDotNet.Running;
43
using BenchmarkDotNet.Configs;
54
using BenchmarkDotNet.Reports;
65
using BenchmarkDotNet.Filters;
76
using BenchmarkDotNet.Jobs;
87
using System.Text;
9-
using System.Runtime;
108
using System.Runtime.InteropServices;
11-
using System.Buffers;
12-
using System.IO;
13-
using System.Collections.Generic;
14-
using System.Linq;
159
using BenchmarkDotNet.Columns;
1610
using System.Runtime.Intrinsics;
1711
using System.Runtime.Intrinsics.X86;
18-
using System.Runtime.Intrinsics.Arm;
19-
using System.Runtime.CompilerServices;
20-
using gfoidl.Base64;
21-
using System.Buffers.Text;
2212

2313
namespace SimdUnicodeBenchmarks
2414
{
2515

26-
16+
#pragma warning disable CA1515
2717
public class Speed : IColumn
2818
{
2919
static long GetDirectorySize(string folderPath)
@@ -76,9 +66,9 @@ public string GetValue(Summary summary, BenchmarkCase benchmarkCase)
7666
public string Legend { get; } = "The speed in gigabytes per second";
7767
}
7868

79-
8069
[SimpleJob(launchCount: 1, warmupCount: 5, iterationCount: 5)]
8170
[Config(typeof(Config))]
71+
#pragma warning disable CA1515
8272
public class RealDataBenchmark
8373
{
8474
#pragma warning disable CA1812
@@ -164,7 +154,8 @@ public Config()
164154
// Parameters and variables for real data
165155
[Params(
166156
@"data/email/",
167-
@"data/dns/swedenzonebase.txt")]
157+
@"data/dns/swedenzonebase.txt"
158+
)]
168159
#pragma warning disable CA1051
169160
public string? FileName;
170161
#pragma warning disable CS8618
@@ -247,7 +238,7 @@ public unsafe void RunScalarDecodingBenchmarkUTF8(string[] data, int[] lengths)
247238
byte[] dataoutput = output[i];
248239
int bytesConsumed = 0;
249240
int bytesWritten = 0;
250-
SimdBase64.Base64.Base64WithWhiteSpaceToBinaryScalar(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
241+
SimdBase64.Scalar.Base64.Base64WithWhiteSpaceToBinaryScalar(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
251242
if (bytesWritten != lengths[i])
252243
{
253244
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
@@ -266,7 +257,7 @@ public unsafe void RunScalarDecodingBenchmarkUTF16(string[] data, int[] lengths)
266257
byte[] dataoutput = output[i];
267258
int bytesConsumed = 0;
268259
int bytesWritten = 0;
269-
SimdBase64.Base64.Base64WithWhiteSpaceToBinaryScalar(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
260+
SimdBase64.Scalar.Base64.Base64WithWhiteSpaceToBinaryScalar(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
270261
if (bytesWritten != lengths[i])
271262
{
272263
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
@@ -285,7 +276,7 @@ public unsafe void RunSSEDecodingBenchmarkUTF8(string[] data, int[] lengths)
285276
byte[] dataoutput = output[i];
286277
int bytesConsumed = 0;
287278
int bytesWritten = 0;
288-
SimdBase64.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
279+
SimdBase64.SSE.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
289280
if (bytesWritten != lengths[i])
290281
{
291282
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
@@ -304,27 +295,27 @@ public unsafe void RunSSEDecodingBenchmarkUTF16(string[] data, int[] lengths)
304295
byte[] dataoutput = output[i];
305296
int bytesConsumed = 0;
306297
int bytesWritten = 0;
307-
SimdBase64.Base64.DecodeFromBase64SSE(base64, dataoutput, out bytesConsumed, out bytesWritten, false);
298+
SimdBase64.SSE.Base64.DecodeFromBase64SSE(base64, dataoutput, out bytesConsumed, out bytesWritten, false);
308299
if (bytesWritten != lengths[i])
309300
{
310301
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
311302
#pragma warning disable CA2201
312303
throw new Exception("Error");
313304
}
314305
}
315-
}
316-
317-
318-
306+
}
307+
308+
309+
319310
public unsafe void RunSSEDecodingBenchmarkWithAllocUTF8(string[] data, int[] lengths)
320311
{
321312
for (int i = 0; i < FileContent.Length; i++)
322313
{
323314
byte[] base64 = input[i];
324-
byte[] dataoutput = new byte[SimdBase64.Base64.MaximalBinaryLengthFromBase64Scalar<byte>(base64.AsSpan())];
315+
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<byte>(base64.AsSpan())];
325316
int bytesConsumed = 0;
326317
int bytesWritten = 0;
327-
SimdBase64.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
318+
SimdBase64.SSE.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
328319
if (bytesWritten != lengths[i])
329320
{
330321
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
@@ -340,10 +331,88 @@ public unsafe void RunSSEDecodingBenchmarkWithAllocUTF16(string[] data, int[] le
340331
{
341332
string s = FileContent[i];
342333
char[] base64 = input16[i];
343-
byte[] dataoutput = new byte[SimdBase64.Base64.MaximalBinaryLengthFromBase64Scalar<char>(base64.AsSpan())];
334+
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<char>(base64.AsSpan())];
335+
int bytesConsumed = 0;
336+
int bytesWritten = 0;
337+
SimdBase64.SSE.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
338+
if (bytesWritten != lengths[i])
339+
{
340+
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
341+
#pragma warning disable CA2201
342+
throw new Exception("Error");
343+
}
344+
}
345+
}
346+
347+
348+
public unsafe void RunARMDecodingBenchmarkUTF8(string[] data, int[] lengths)
349+
{
350+
for (int i = 0; i < FileContent.Length; i++)
351+
{
352+
//string s = FileContent[i];
353+
byte[] base64 = input[i];
354+
byte[] dataoutput = output[i];
355+
int bytesConsumed = 0;
356+
int bytesWritten = 0;
357+
SimdBase64.Arm.Base64.DecodeFromBase64ARM(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
358+
if (bytesWritten != lengths[i])
359+
{
360+
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
361+
#pragma warning disable CA2201
362+
throw new Exception("Error");
363+
}
364+
}
365+
}
366+
367+
public unsafe void RunARMDecodingBenchmarkUTF16(string[] data, int[] lengths)
368+
{
369+
for (int i = 0; i < FileContent.Length; i++)
370+
{
371+
string s = FileContent[i];
372+
ReadOnlySpan<char> base64 = s.AsSpan();
373+
byte[] dataoutput = output[i];
374+
int bytesConsumed = 0;
375+
int bytesWritten = 0;
376+
SimdBase64.Arm.Base64.DecodeFromBase64ARM(base64, dataoutput, out bytesConsumed, out bytesWritten, false);
377+
if (bytesWritten != lengths[i])
378+
{
379+
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
380+
#pragma warning disable CA2201
381+
throw new Exception("Error");
382+
}
383+
}
384+
}
385+
386+
387+
388+
public unsafe void RunARMDecodingBenchmarkWithAllocUTF8(string[] data, int[] lengths)
389+
{
390+
for (int i = 0; i < FileContent.Length; i++)
391+
{
392+
byte[] base64 = input[i];
393+
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<byte>(base64.AsSpan())];
394+
int bytesConsumed = 0;
395+
int bytesWritten = 0;
396+
SimdBase64.Arm.Base64.DecodeFromBase64ARM(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
397+
if (bytesWritten != lengths[i])
398+
{
399+
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
400+
#pragma warning disable CA2201
401+
throw new Exception("Error");
402+
}
403+
}
404+
}
405+
406+
public unsafe void RunARMDecodingBenchmarkWithAllocUTF16(string[] data, int[] lengths)
407+
{
408+
for (int i = 0; i < FileContent.Length; i++)
409+
{
410+
string s = FileContent[i];
411+
char[] base64 = input16[i];
412+
byte[] dataoutput = new byte[SimdBase64.Scalar.Base64.MaximalBinaryLengthFromBase64Scalar<char>(base64.AsSpan())];
344413
int bytesConsumed = 0;
345414
int bytesWritten = 0;
346-
SimdBase64.Base64.DecodeFromBase64SSE(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
415+
SimdBase64.Arm.Base64.DecodeFromBase64ARM(base64.AsSpan(), dataoutput, out bytesConsumed, out bytesWritten, false);
347416
if (bytesWritten != lengths[i])
348417
{
349418
Console.WriteLine($"Error: {bytesWritten} != {lengths[i]}");
@@ -419,23 +488,6 @@ public unsafe void DotnetRuntimeBase64RealDataUTF16()
419488
RunRuntimeDecodingBenchmarkUTF16(FileContent, DecodedLengths);
420489
}
421490

422-
// Gfoidl does not work correctly with spaces.
423-
/*[Benchmark]
424-
[BenchmarkCategory("default", "gfoidl")]
425-
public unsafe void DotnetGfoildBase64RealDataUTF16()
426-
{
427-
RunGfoidlDecodingBenchmarkUTF16(FileContent, DecodedLengths);
428-
}*/
429-
430-
// We almost never want to benchmark scalar decoding.
431-
/*[Benchmark]
432-
[BenchmarkCategory("scalar")]
433-
public unsafe void ScalarDecodingRealDataUTF8()
434-
{
435-
RunScalarDecodingBenchmarkUTF8(FileContent, DecodedLengths);
436-
}*/
437-
438-
439491
[Benchmark]
440492
[BenchmarkCategory("SSE")]
441493
public unsafe void SSEDecodingRealDataUTF8()
@@ -450,6 +502,27 @@ public unsafe void SSEDecodingRealDataWithAllocUTF8()
450502
RunSSEDecodingBenchmarkWithAllocUTF8(FileContent, DecodedLengths);
451503
}
452504

505+
[Benchmark]
506+
[BenchmarkCategory("arm64")]
507+
public unsafe void ARMDecodingRealDataUTF8()
508+
{
509+
RunARMDecodingBenchmarkUTF8(FileContent, DecodedLengths);
510+
}
511+
512+
[Benchmark]
513+
[BenchmarkCategory("arm64")]
514+
public unsafe void ARMDecodingRealDataWithAllocUTF8()
515+
{
516+
RunARMDecodingBenchmarkWithAllocUTF8(FileContent, DecodedLengths);
517+
}
518+
519+
[Benchmark]
520+
[BenchmarkCategory("arm64")]
521+
public unsafe void ARMDecodingRealDataUTF16()
522+
{
523+
RunARMDecodingBenchmarkUTF16(FileContent, DecodedLengths);
524+
}
525+
453526
[Benchmark]
454527
[BenchmarkCategory("SSE")]
455528
public unsafe void SSEDecodingRealDataUTF16()
@@ -463,8 +536,8 @@ public unsafe void SSEDecodingRealDataWithAllocUTF16()
463536
{
464537
RunSSEDecodingBenchmarkWithAllocUTF16(FileContent, DecodedLengths);
465538
}
466-
467539
}
540+
#pragma warning disable CA1515
468541
public class Program
469542
{
470543
static void Main(string[] args)

‎benchmark/benchmark.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net8.0</TargetFramework>
5+
<TargetFramework>net9.0</TargetFramework>
66
<ImplicitUsings>enable</ImplicitUsings>
77
<Nullable>enable</Nullable>
88
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>

0 commit comments

Comments
 (0)