Tuesday, 3 November 2020

How to parse items from a large JSON stream in JavaScript?

So I have downloaded the Wikidata JSON dump and it's about 90GB, too large to load into memory. It consists of a simple JSON structure like this:

[
  item,
  item,
  item,
  ...
]

Each "item" looks something like this:

{
  "type": "item",
  "id": "Q23",
  "labels": {
    "<lang>": obj
  },
  "descriptions": {
    "<lang>": {
      "language": "<lang>",
      "value": "<string>"
    },
  },
  "aliases": {
    "<key>": [
      obj,
      obj,
    ],
  },
  "claims": {
    "<keyID>": [
       {
        "mainsnak": {
          "snaktype": "value",
          "property": "<keyID>",
          "datavalue": {
            "value": {
              "entity-type": "<type>",
              "numeric-id": <num>,
              "id": "<id>"
            },
            "type": "wikibase-entityid"
          },
          "datatype": "wikibase-item"
        },
        "type": "statement",
        "id": "<anotherId>",
        "rank": "preferred",
        "references": [
          {
            "hash": "<hash>",
            "snaks": {
              "<keyIDX>": [
                {
                  "snaktype": "value",
                  "property": "P854",
                  "datavalue": obj,
                  "datatype": "url"
                }
              ]
            },
            "snaks-order": [
              "<propID>"
            ]
          }
        ]
      }
    ]
  },
  "sitelinks": {
    "<lang>wiki": {
      "site": "<lang>wiki",
      "title": "<string>",
      "badges": []
    }
  }
}

The JSON stream is configured like this:

const fs   = require('fs')
const zlib = require('zlib')
const { parser } = require('stream-json')

let stream = fs.createReadStream('./wikidata/latest-all.json.gz')
stream
  .pipe(zlib.createGunzip())
  .pipe(parser())
  .on('data', buildItem)

function buildItem(data) {
  switch (data.name) {
    case `startArray`:
      break
    case `startObject`:
      break
    case `startKey`:
      break
    case `stringChunk`:
      break
    case `endKey`:
      break
    case `keyValue`:
      break
    case `startString`:
      break
    case `endString`:
      break
    case `stringValue`:
      break
    case `endObject`:
      break
    case `endArray`:
      break
  }
}

Notice the buildItem has the key information, it shows that the JSON stream emits objects like this (these are the logs):

{ name: 'startArray' }
{ name: 'startObject' }
{ name: 'startKey' }
{ name: 'startString' }
{ name: 'stringValue', value: 'type' }
{ name: 'endString' }
...

How do you parse this into item objects like the above? Parsing this linear stream into a tree is very difficult to comprehend.

A sample of output from the JSON stream is here, which you could use to test a parser if it helps.



from How to parse items from a large JSON stream in JavaScript?

No comments:

Post a Comment