-
Notifications
You must be signed in to change notification settings - Fork 226
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1012 from fiatjaf/negentropy
Add Negentropy format
- Loading branch information
Showing
13 changed files
with
3,297 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| package negentropy | ||
|
|
||
| // https://github.com/hoytech/negentropy | ||
|
|
||
| import ( | ||
| "embed" | ||
| "math" | ||
| "time" | ||
|
|
||
| "github.com/wader/fq/format" | ||
| "github.com/wader/fq/pkg/decode" | ||
| "github.com/wader/fq/pkg/interp" | ||
| "github.com/wader/fq/pkg/scalar" | ||
| ) | ||
|
|
||
| //go:embed negentropy.md | ||
| var negFS embed.FS | ||
|
|
||
| func init() { | ||
| interp.RegisterFormat( | ||
| format.Negentropy, | ||
| &decode.Format{ | ||
| Description: "Negentropy message", | ||
| DecodeFn: decodeNegentropyMessage, | ||
| Groups: []*decode.Group{}, | ||
| }) | ||
| interp.RegisterFS(negFS) | ||
| } | ||
|
|
||
| const ( | ||
| version = 0x61 | ||
| fingerprintSize = 16 | ||
|
|
||
| modeSkip = 0 | ||
| modeFingerprint = 1 | ||
| modeIdlist = 2 | ||
| ) | ||
|
|
||
| var modeMapper = scalar.SintMapSymStr{ | ||
| modeSkip: "skip", | ||
| modeFingerprint: "fingerprint", | ||
| modeIdlist: "idlist", | ||
| } | ||
|
|
||
| type timestampDeltaTranslator struct{} | ||
|
|
||
| func (tt *timestampDeltaTranslator) MapSint(s scalar.Sint) (scalar.Sint, error) { | ||
| if s.Actual == 0 { | ||
| s.Sym = -1 | ||
| s.Description = "infinity" | ||
| return s, nil | ||
| } else { | ||
| s.Sym = s.Actual - 1 | ||
| return s, nil | ||
| } | ||
| } | ||
|
|
||
| type timestampTranslator struct{ last time.Time } | ||
|
|
||
| func (tt *timestampTranslator) MapSint(s scalar.Sint) (scalar.Sint, error) { | ||
| if s.Actual == 0 { | ||
| s.Description = "infinity" | ||
| tt.last = time.Unix(math.MaxInt64, 0) | ||
| return s, nil | ||
| } else { | ||
| timestamp := tt.last.Add(time.Second * time.Duration(s.Actual-1)) | ||
| s.Description = timestamp.UTC().Format(time.RFC3339) | ||
| s.Actual = timestamp.Unix() | ||
| tt.last = timestamp | ||
| return s, nil | ||
| } | ||
| } | ||
|
|
||
| func decodeNegentropyMessage(d *decode.D) any { | ||
| tdt := ×tampDeltaTranslator{} | ||
| tt := ×tampTranslator{last: time.Unix(0, 0)} | ||
|
|
||
| d.Endian = decode.BigEndian | ||
|
|
||
| v := d.FieldU8("version") | ||
| if v != version { | ||
| d.Fatalf("unexpected version %d (expected %d), is this really a negentropy message?", v, version) | ||
| } | ||
|
|
||
| d.FieldStructArrayLoop("bounds", "bound", d.NotEnd, func(d *decode.D) { | ||
| delta := d.FieldSintFn("timestamp_delta", decodeVarInt, tdt) | ||
| d.FieldValueSint("timestamp", delta, tt) | ||
|
|
||
| size := d.FieldSintFn("id_prefix_size", decodeVarInt) | ||
| if size > 32 { | ||
| d.Fatalf("unexpected id prefix size bigger than 32: %d", size) | ||
| } | ||
| if size > 0 { | ||
| d.FieldRawLen("id_prefix", size*8, scalar.RawHex) | ||
| } | ||
|
|
||
| mode := d.FieldSintFn("mode", decodeVarInt, modeMapper) | ||
| switch mode { | ||
| case modeSkip: | ||
| return | ||
| case modeFingerprint: | ||
| d.FieldRawLen("fingerprint", fingerprintSize*8, scalar.RawHex) | ||
| return | ||
| case modeIdlist: | ||
| d.FieldStruct("idlist", func(d *decode.D) { | ||
| num := d.FieldSintFn("num", decodeVarInt) | ||
| d.FieldArray("ids", func(d *decode.D) { | ||
| for i := 0; i < int(num); i++ { | ||
| d.FieldRawLen("id", 32*8, scalar.RawHex) | ||
| } | ||
| }) | ||
| }) | ||
| default: | ||
| d.Fatalf("unexpected mode %d", mode) | ||
| } | ||
| }) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func decodeVarInt(d *decode.D) int64 { | ||
| res := 0 | ||
| for { | ||
| b := int(d.U8()) | ||
| res = (res << 7) | (b & 127) | ||
| if (b & 128) == 0 { | ||
| break | ||
| } | ||
| } | ||
| return int64(res) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| ### View a full Negentropy message | ||
|
|
||
| ``` | ||
| $ fq -d negentropy dd file | ||
| ``` | ||
|
|
||
| ### Or from hex | ||
|
|
||
| ``` | ||
| $ echo '6186b7abb47c0001108e4206828ee3bf34258465809a337c6c00019a68e37b177a50b3ae7164ccc628b962020114019c1381281c9e3849d5fbd514b7bb65ad0101e601fbf7451f5d22e7fa36ae3e910e9f5215020157014a1b26853e06e9c32eb41b1df4f9ab300201e6011840e273c84bb1344f1d4e15d9aa67920200016f12ee2340888653f10b0ec2d438ac9f0101840156d2d796f4dff004ab369b9bcfa4d81e020187013f1b3c8a019800d5764e2de6bdfd2785020114017caaf0acb5dfe249aa0f7f742402168a01018301e7b8c4decb1eae455ca5714281e3245302017a01409c22636b097362df125ddffb6d944302015b01f332208bee82acf8ed922853ee54057f020001fc3e51fdb0b92966e38017f7959903850101cc01428ce0c96d49f15b50143e4fb228cb9300000131712d30e5296a7a45d07bba452d61cd' | fq -R 'from_hex | negentropy | dd' | ||
| ``` | ||
|
|
||
| ### Check how many ranges the message has and how many of those are of 'fingerprint' mode | ||
|
|
||
| ``` | ||
| $ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message | ||
| ``` | ||
|
|
||
| ### Check get all ids in all idlists | ||
|
|
||
| ``` | ||
| $ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message | ||
| ``` | ||
|
|
||
| ### Authors | ||
| - fiatjaf, https://fiatjaf.com | ||
|
|
||
| ### References | ||
| - https://github.com/hoytech/negentropy |
Binary file not shown.
Oops, something went wrong.