term-vector

A node.js module that creates a term vector from tokenized text. Use term-vector when implementing a vector space model

Works with Unicode!

Does ngrams!

1
2const tv = require('term-vector') 
3// alternatively if you are all fancy and new-fangled:
4// import tv from 'term-vector'
5const tokens = 'this is really really really cool'.split(' ')
6
7// just make a simple term vector
8tv(tokens)
9// [
10//   { term: [ 'cool' ], positions: [ 5 ] },
11//   { term: [ 'is' ], positions: [ 1 ] },
12//   { term: [ 'really' ], positions: [ 2, 3, 4 ] },
13//   { term: [ 'this' ], positions: [ 0 ] }
14// ]
15
16// make a term vector with ngrams of length 1 and 2
17tv(tokens, { ngramLengths: [ 1, 2 ] })
18// [
19//   { term: [ 'cool' ], positions: [ 5 ] },
20//   { term: [ 'is' ], positions: [ 1 ] },
21//   { term: [ 'is', 'really' ], positions: [ 1 ] },
22//   { term: [ 'really' ], positions: [ 2, 3, 4 ] },
23//   { term: [ 'really', 'really' ], positions: [ 2, 3 ] },
24//   { term: [ 'really', 'cool' ], positions: [ 4 ] },
25//   { term: [ 'this' ], positions: [ 0 ] },
26//   { term: [ 'this', 'is' ], positions: [ 0 ] }
27// ]
28