typed-struct

A JavaScript utility library (written in TypeScript) for creating objects and classes that store their properties in a buffer for serialization/deserialization similar to structures in C.

Getting started

Using npm:

1$ npm install typed-struct

or yarn:

1$ yarn add typed-struct

To use the module in a browser environment, add the buffer package to the dependencies.

1npm install buffer

Features

The following types are supported:

• Integer (1, 2, 4, 8 bytes)
• Boolean (1, 2, 4 bytes)
• Float (4 bytes)
• Double (8 bytes)
• Bit fields (1, 2, 4 bytes)
• Binary-decoded decimal (1 byte)
• Buffer
• String (with iconv-lite encodings)
• Custom type (requires custom getter/setter)
• Works in node.js and the browser (requires buffer for the browser)

Fixed values, endianness, nested types and arrays are also supported.

The generated structures will be strongly typed, which will provide better documentation and allow TypeScript to validate that your code is working correctly.

API

Examples

Create a data structure by chaining the appropriate method calls

1const { Struct } = require('typed-struct');
2// or
3import { Struct } from 'typed-struct';
4
5const MyStructure = new Struct('MyStructure') // give a name to the constructor
6  .Int8('foo')        // signed 8-bit integer field `foo`
7  .UInt16LE('bar')    // unsigned, little-endian 16-bit integer field `bar`
8  .compile();         // create a constructor for the structure, called last
9
10// You can use the compiled constructor as if you had made a class.
11// If you are using a typescript, you will also get
12// the exact type of the generated structure
13const item1 = new MyStructure();      // creates an instance
14expect(item1.constructor.name).toBe('MyStructure');
15
16// This class has useful static properties and methods.
17const raw = MyStructure.raw(item1);   // get the underlying buffer
18expect(MyStructure.baseSize).toBe(3); // the minimum base size of the structure
19expect(raw.length).toBe(MyStructure.baseSize);
20
21// Changing the properties of the structure
22// changes the underlying buffer
23item1.foo = 0x56;
24item1.bar = 0x1234;
25expect(raw).toEqual(Buffer.from([0x56, 0x34, 0x12]))
26
27// ... and vice versa
28raw[0] = 0;
29expect(item1.foo).toBe(0);
30
31// deserialization
32const item2 = new MyStructure([0x11, 0x22, 0x33]);
33expect(item2.foo).toBe(0x11);
34expect(item2.bar).toBe(0x3322);

Typed arrays

1import { Struct } from 'typed-struct';
2
3const Foo = new Struct('Foo')
4  .UInt16Array('items', 10)
5  .compile();
6
7const foo = new Foo();
8expect(foo.items).toHaveLength(10);
9expect(foo.items).toBeInstanceOf(Uint16Array);
10expect(Foo.raw(foo)).toHaveLength(20);
11
12// If the byte order in the system is different from that used in the data,
13// use `swap` method
14Foo.swap(foo, 'items');
15
16// ...
17
18// change back
19Foo.swap(foo, 'items');

Array of structures

Using the structure MyStructure from the previous examples

1const Baz = new Struct('Baz')
2  .StructArray('structs', MyStructure, 10)
3  .compile();
4
5const baz = new Baz();
6expect(baz.structs).toHaveLength(10);
7baz.structs[3].foo = 123;
8// but
9expect(() => {
10  baz.struct[3] = new MyStructure();
11}).toThrow(new TypeError('Cannot assign to read only property "3"'))
12
13expect(() => {
14  baz.structs.push(new MyStructure());
15}).toThrow(new TypedError('Cannot add property 10, object is not extensible'));
16

Nested types

Using the structure MyStructure from the previous examples

1const Bat = new Struct('Bat')
2  .Float64LE('baz')
3  .Struct('quux', MyStructure)
4  .compile();
5
6const bat = new Bat();
7bat.quux.foo = 123;
8bat.quux.bar = 82345;
9bat.baz = 3.14;
10
11const Xyz = new Struct('Xyz')
12  .Struct('bat', Bat)
13  .compile();
14
15const xyz = new Xyz();
16xyz.bat.quux.foo = 123;
17
18// but assignment to structure field is not allowed
19expect(() => {
20  xyz.bat = bat;
21}).toThrow(new TypeError('Cannot assign to read only property bat'))

Unions

1const Word = new Struct('Word')
2  .UInt16LE('value')
3  // We take a step back or two bytes back, which the previous field takes up
4  // you can also use `seek` method with a negative argument
5  .back()
6  .UInt8('low')
7  .UInt8('high')
8  .compile();
9
10const word = new Word();
11word.value = 0x1234;
12expect(word.low).toBe(0x34);
13expect(word.high).toBe(0x12);
14expect(Word.baseSize).toBe(2);

Custom types

1const BirthCertificate = new Struct('BirthCertificate')
2  .Custom(
3    'birthday',
4    8, // size
5    // getter, must return property value or `undefined` for unknown type
6    (type, buf) => new Date(buf.readDoubleLE() * 1000),
7    // setter, must return `false` for unknown type or `true` if everything is ok
8    (type, buf, value) => buf.writeDoubleLE(value.getTime() / 1000) > 0
9  )
10  .compile();
11expect(BirthCertificate.baseSize).toBe(8);
12
13const cert = new BirthCertificate();
14cert.birthday = new Date(1973, 6, 15);

Aliases

Using the structure BirthCertificate from the previous example.

1const Twins = new Struct('Twins')
2  // `John` and `Jimmy` properties are equivalent
3  .Struct(['John', 'Jimmy'], BirthCertificate)
4  .compile();
5
6expect(Twins.baseSize).toBe(8);
7
8const twins = new Twins();
9twins.John.birthday = new Date(1973, 6, 15);
10expect(twins.Jimmy.birthday).toBe(twins.John.birthday);

Fixed values

1const Request = new Struct('Request')
2  .UInt32BE('header', 0xDEADBEEF)
3  .Buffer('data', 16)
4  .compile();
5
6const req = new Request();
7
8// `header` property is initialized in the constructor
9expect(req.header).toBe(0xDEADBEEF);
10
11// Assigning a value other than the specified one is not allowed
12expect(() => {
13  // eslint or IDE will also indicate an error
14  req.header = 0xDEADC0DE;
15}).toThrow(new TypeError(`Invalid value, expected 0xDEADBEEF`));
16
17// ... but
18req.header = 0xDEADBEEF;

Enumerated fields (Typescript)

1import { Struct, typed } from 'typed-struct';
2
3enum ErrorType {
4  Success,
5  Timeout,
6  InvalidCommand,
7  ServerError,
8}
9
10const REQUEST = 0xAA;
11const RESPONSE = 0x55;
12
13// Numeric type refinement
14const OperationResult = new Struct()
15  .UInt8('operation', typed<1 | 2 | 3 | 19>())
16  .UInt8('type', typed<typeof REQUEST | typeof RESPONSE>())
17  .UInt8('error', typed<ErrorType>())
18  .compile();
19
20/*
21type OperationResult = {
22  operation: 1 | 2 | 3 | 19;
23  type: 0xAA | 0x55;
24  error: ErrorType;
25}
26*/
27
28const res = new OperationResult();
29res.error = ErrorType.Success;

Variable length structures

1const Package = new Struct('Package')
2  .UInt16LE('length')
3  // If you do not specify the length of the buffer,
4  // it will take all the remaining space.
5  // There can only be one such field, and it must be the last.
6  // You can specify a negative buffer length (-N),
7  // then the buffer will take up all space except for the last N bytes
8  // Since the size of the buffer is unknown, `baseSize` ignores its length.
9  .Buffer('data')
10  .compile();
11expect(Package.baseSize).toBe(2);
12
13function createPackage(data: Buffer): Package {
14  const pkg = new Package(Package.baseSize + data.length);
15  data.copy(pkg.data);
16  pkg.length = data.length;
17  return pkg;
18}

CRC

The checksum is often used to verify data integrity, it is usually stored in the last bytes of the buffer.

1const Exact = new Struct('Exact')
2  .UInt16LE('length')
3  .Buffer('data')
4  // Only the checksum can go after the tail buffer
5  .CRC16LE('crc')
6  .compile();
7expect(Exact.baseSize).toBe(4);

If you pass the checksum function as parameters, then your structure will have the static crc method that you can use to calculate and update this field.

Let's slightly modify the previous example:

1import { crc16 } from 'crc';
2
3const Exact = new Struct('Exact')
4  .UInt16LE('length')
5  .Buffer('data')
6  // checksum function and initial value
7  .CRC16LE('crc', crc16, 0xffff)
8  .compile();
9
10function createExact(data: Buffer): Exact {
11  const item = new Exact(Exact.baseSize + data.length);
12  data.copy(item.data);
13  item.length = data.length;
14  // New static method `Exact.crc`
15  item.crc = Exact.crc(item);
16  // or the same thing - calculate and update the field
17  Exact.crc(item, true);
18  return item;
19}

Strings

1const Person = new Struct('Person')
2  .String('name', 30)
3  .String('surname', 40)
4  .compile();
5
6expect(Person.baseSize).toBe(70);
7
8const walterGreen = new Person();
9walterGreen.name = 'Walter';
10walterGreen.surname = 'Green';

You can specify the encoding, the default is UTF8. You can use the encodings supported by Buffer or install iconv-lite package to add support for all encodings defined in this package.

1const Foo = new Struct('Foo')
2  // Specify the desired encoding and size in bytes like this
3  .String('bar', 'win1251', 10)
4  // or so
5  .String('baz', 10, 'ucs2')
6  // or even so
7  .String('quux', { length: 10, encoding: 'koi8-r', literal: 'Андрей' })
8  .compile();

If you do not specify the length, then the string field will take up all the remaining space.

String Arrays

1const Page = new Struct('Page')
2  .StringArray('body', {
3    length: 80,        // string length in bytes (required)
4    lines: 25,         // number of lines (required)
5    encoding: 'ascii'
6  })
7  .compile();
8
9const page = new Page();
10expect(Page.baseSize).toBe(80 * 25);
11expect(Array.isArray(page.body)).toBe(true);
12const body = page.body;
13body[0] = 'Lorem ipsum';
14expect(page.body[0]).toBe(body[0]);
15const raw = Page.raw(page);
16raw[Page.getOffsetOf('body')] = 'l'.charCodeAt(0);
17expect(body[0]).toBe('lorem ipsum');
18console.log(page);

output

Page {
  body: [
    'lorem ipsum', '', '',
    '',            '', '',
    '',            '', '',
    '',            '', '',
    '',            '', '',
    '',            '', '',
    '',            '', '',
    '',            '', '',
    ''
  ]
}

Bit fields

1// |------> bit order from the most significant bits to the least significant
2// 01234567 offset
3// 00011110 mask
4//    \  /
5//     \/
6// bit field with an offset of 3 bits and a length of 4 bits
7const Foo = new Struct('Foo')
8  .Bits8({
9    // for each property, specify the offset and length
10    high: [0, 4],
11    low: [4, 4],
12    // properties may overlap
13    value: [0, 8]
14  })
15  .compile();
16expect(Foo.baseSize).toBe(1);
17const foo = new Foo();
18foo.value = 0x36;
19expect(foo.high).toBe(3);
20expect(foo.low).toBe(6);

Reserved fields

1const Bar = new Struct('Bar')
2  .Int16LE('baz')
3  // Skip two bytes.
4  // You can use negative values for unions.
5  // Zero moves the current pointer to the end.
6  .seek(2)
7  .compile();
8
9expect(Bar.baseSize).toBe(4);  

JSON

e.g.

1const Model = new Struct('Model')
2  .Int8('foo')
3  .UInt8Array('bars', 4)
4  .Struct('nested', new Struct('Nested').Int8('value').Buffer('items', 4).compile())
5  .compile();
6
7const model = new Model([0x10, 1, 2, 3, 4, 0x20, 5, 6, 7, 8]);

Let's see what the model is ^[note*]

1console.log(model);

output: ($raw is a hidden property and is shown for clarity)

Model {
  foo: [Getter/Setter],
  bars: Uint8Array(4) [ 1, 2, 3, 4 ],
  nested: Nested {
    value: [Getter/Setter],
    items: <Buffer 05 06 07 08>
  }
  $raw: <Buffer 10 01 02 03 04 20 05 06 07 08>
}

If you only need a parser, you can avoid overheads like getters and setters and hidden buffers by using the method toJSON. The result contains only data, as opposed to methods or internal state. Its prototype is Object.prototype and all numeric iterable types (buffer, typed arrays) are replaced with number[], all custom types other than number, boolean, string, string[] and numeric iterables are replaced with the result of executing toJSON or toString methods.

1console.log(model.toJSON());

output:

{
  foo: 16,
  bars: [ 1, 2, 3, 4 ],
  nested: { value: 32, items: [ 5, 6, 7, 8 ] }
}

Note:

Starting from version 2.3.0 in the environment of node.js, the expression console.log(model) behaves the same as console.log(model.toJSON()). This is done to make debugging easier.

toString()

For ease of debugging, the generated structures have an overridden toString method that outputs a string of hexadecimal bytes separated by an equals sign at field boundaries. If there is a debug package in the dependencies, then these fields will be colored depending on the field name.

console.log(`${model}`);

01=ce-ca-23-00-00-00-00-00=ff-ff-ff-ff-ff-ff

A typical example of working with binary data via a serial port

Package.ts

1import { Struct, typed, type ExtractType } from 'typed-struct';
2import { crc16 } from 'crc';
3
4export enum Command {
5  Read,
6  Write,
7  Reset,
8  Halt,
9}
10
11export const PREAMBLE = 0x1234;
12
13/**
14 * Variable length structure 
15 * Package.baseSize = 9 - minimal structure size
16 */
17export const Package = new Struct('Package')
18  .UInt16BE('header', PREAMBLE)
19  .UInt8('source')
20  .UInt8('destination')
21  .UInt8('command', typed<Command>())
22  .UInt16LE('length')
23  .Buffer('data')
24  .CRC16LE('crc', crc16)
25  .compile();
26
27// it's not obligatory
28export type Package = ExtractType<typeof Package>;
29/*
30type Package = {
31  header: 0x1234;
32  source: number;
33  destination: number;
34  command: Command;
35  length: number;
36  data: Buffer;
37  crc: number;
38}
39 */

Decoder.ts

1import {
2  Transform,
3  TransformOptions,
4  TransformCallback
5} from 'stream';
6
7import { Package, PREAMBLE } from './Package';
8
9const preamble = Buffer.alloc(2);
10preamble.writeInt16BE(PREAMBLE);
11
12const empty = Buffer.alloc(0);
13
14const lengthOffset = Package.getOffsetOf('length');
15
16export default class Decoder extends Transform {
17  private buf = empty;
18
19  constructor(options?: TransformOptions) {
20    super({
21      ...options,
22      readableObjectMode: true,
23    });
24  }
25
26  _transform(chunk: unknown, encoding: BufferEncoding, callback: TransformCallback) {
27    if (Buffer.isBuffer(chunk)) {
28      const data = Buffer.concat([this.buf, chunk]);
29      if (data.length > 0) {
30        this.buf = this.recognize(data);
31      }
32    }
33    callback();
34  }
35  
36  _flush(callback: TransformCallback) {
37    this.buf = empty;
38    callback();
39  }
40
41  private recognize(data: Buffer): Buffer {
42    for (let offset = 0;;) {
43      const rest = data.length - offset;
44      if (rest <= 0) return empty;
45      const start = data.indexOf(rest < preamble.length ? preamble.slice(0, rest) : preamble, offset);
46      if (start === -1) return empty;
47      const frame = data.slice(start);
48      if (frame.length < Package.baseSize) return frame;
49      const length = frame.readUInt16LE(lengthOffset);
50      if (length <= MAX_LENGTH) {
51
52        // calculate the total size of structure
53        const total = length + Package.baseSize;
54        if (frame.length < total) return frame;
55
56        // deserialize Package from the buffer
57        const pkg = new Package(frame.slice(0, total));
58
59        // crc check
60        if (Package.crc(pkg) === pkg.crc) {
61
62          // push decoded package
63          this.push(pkg);
64          offset = start + total;
65        }
66      }
67      
68      if (offset <= start) {
69        offset = start + 1;
70      }
71    }
72  }
73}

Encoder.ts

1import { Transform, TransformCallback, TransformOptions } from 'stream';
2
3import { Package } from './Package';
4
5export default class Encoder extends Transform {
6  constructor(options?: TransformOptions) {
7    super({
8      ...options,
9      writableObjectMode: true,
10    });
11  }
12
13  public _transform(chunk: unknown, encoding: BufferEncoding, callback: TransformCallback): void {
14    const chunks = Array.isArray(chunk) ? chunk : [chunk];
15    chunks.forEach(pkg => {
16      
17      // instance type check
18      if (pkg instanceof Package) {
19        
20        // getting a raw buffer
21        const raw = Package.raw(pkg);
22
23        // update length
24        pkg.length = pkg.data.length;
25
26        // update crc
27        Package.crc(pkg, true);
28        
29        // serialize
30        this.push(raw);
31      }
32    });
33    callback();
34  }
35}

serial.ts

1import pump from 'pump';
2import SerialPort from 'serialport';
3import Decoder from './Decoder';
4import Encoder from './Encoder';
5import { Package, Command } from './Package';
6
7// ...
8
9const decoder = new Decoder();
10const encoder = new Encoder();
11let connected = false;
12
13const serial = new SerialPort(path, { baudRate: 115200 }, err => {
14  if (err) {
15    console.error(`error while open ${path}`)
16  } else {
17    // create a pipeline
18    pump(encoder, serial, decoder, err => {
19      connected = false;
20      console.log('pipe finished', err);
21    });
22    connected = true;
23  }
24});
25
26decoder.on('data', (res: Package) => {
27  // Processing the package
28});
29
30// Example function for sending data
31async function send(data: Buffer, src: number, dest: number): Promise<void> {
32  if (!connected) throw new Error('Connection closed');
33  
34  // Create a packet with the specified buffer size
35  const pkg = new Package(data.length + Package.baseSize);
36  
37  // Package initialization
38  data.copy(pkg.data);
39  pkg.source = src;
40  pkg.destination = dest;
41  pkg.command = Command.Write;
42  
43  // Sending a package
44  if (!encoder.write(pkg)) {
45    await new Promise(resolve => encoder.once('drain', resolve));
46  }
47}
48
49const release = () => {
50  serial.isOpen && serial.close();
51}
52process.on('SIGINT', release);
53process.on('SIGTERM', release);

License

This project is licensed under the MIT License - see the LICENSE file for details