Dataframes

The @sluice/nanoarrow-js package implements a high-performance dataframe library for javascript environments.

It includes a set of helper functions to interpret and generate Arrow C Data Interface and Arrow C Stream Interface structures.

It allows fast no-copy memory sharing between the javascript runtime and native (FFI or WASM) bindings.

Whereas the current suite of Arrow implementations provide the basis for a comprehensive data analysis toolkit, this library is intended to support clients that wish to produce or interpret Arrow C Data and/or Arrow C Stream structures where linking to a higher level Arrow binding is not preferred. This library has zero dependencies.

Usage

import {
  ArrowArrayJs,
  ArrowArrayStruct,
  ArrowDictionary,
  ArrowNanoType,
  ArrowSchemaJs,
  ArrowStreamJs,
} from '@sluice/nanoarrow-js';

Constructors

Dataframe

  const insertValues = new ArrowArrayStruct(
      { a: ['hello', 'world', null], m: ['sad', 'happy', 'happy'] },
      { a: ArrowNanoType.String, m: mood },
    );

Rows

const insertValues = new ArrowArrayStruct(
  [
    { a: 'hello', m: 'sad' },
    { a: 'world', m: 'happy' },
  ],
  { a: ArrowNanoType.String, m: mood },
);

Array of Columns

const insertValues = new ArrowArrayStruct(
  [
    ['hello', 'world'],
    ['sad', 'happy'],
  ],
  { a: ArrowNanoType.String, m: mood },
);

Enums

enum Mood {
  happy = 0,
  ok,
  sad,
}

let mood: ArrowDictionary;
mood = new ArrowDictionary('mood', ArrowNanoType.Int8, ArrowNanoType.String, Mood);

Introspect schema and data

Schema Detail

TypeScript

const schema = insertValues.schema.toPortable();

Result

{
  type: 'struct',
  children: [
    {
      name: 'a',
      type: 'string',
    },
    {
      name: 'm',
      type: 'int8',
      dictionary: {
        name: 'mood',
        indexType: 'int8',
        valueType: 'string',
      },
    },
  ],
}

Schema Summary

TypeScript

const schemaStr = insertValues.schema.toPortable();

Result

'struct {a:string, m:int8 (dictionary mood string)}'

Data as Rows

TypeScript

const rows = insertValues.rows();

Result

[
  {
    a: 'hello',
    m: 'sad',
  },
  {
    a: 'world',
    m: 'happy',
  },
  {
    a: null,
    m: 'happy',
  },
]

Data as Array of arrays

TypeScript

const arrayofarrays = insertValues.values();

Result

[
  ['hello', 'world', null],
  ['sad', 'happy', 'happy'],
]

Data as Dataframe

TypeScript

const dataframe = insertValues.dfPlain();

Result

{
  a: ['hello', 'world', null],
  m: ['sad', 'happy', 'happy'],
}

C Interop

C Interface Export

const insertValues = new ArrowArrayStruct(
    { a: ['hello', 'world', null], m: ['sad', 'happy', 'happy'] },
    { a: ArrowNanoType.String, m: mood },
  );

  expect(insertValues.schema.toString()).toStrictEqual('struct {a:string, m:int8 (dictionary mood string)}');
  expect(insertValues.rows()).toStrictEqual([
    {
      a: 'hello',
      m: 'sad',
    },
    {
      a: 'world',
      m: 'happy',
    },
    {
      a: null,
      m: 'happy',
    },
  ]);

  const c = insertValues.toFFI(ffi.memoryInterface);

C Interface import

//  const ptrSchema = insertValues.schema.toFFI(ffi.memoryInterface);
//  const ptrArray = insertValues.toFFI(ffi.memoryInterface);
const viewSchema = ArrowSchemaJs.fromFFI(ptrSchema, ffi.memoryInterface, insertValues.schema.release);

Arrow Streams

const insertValues = new ArrowArrayStruct(
  { a: ['hello', 'world', null], m: ['sad', 'happy', 'happy'] },
  { a: ArrowNanoType.String, m: mood },
);

const stream = ArrowStreamJs.fromArray(insertValues);
const arrayGenerator = stream.stream();
const arrayFirst = arrayGenerator.next();
const arraySecond = arrayGenerator.next();
// ...