So I have downloaded the Wikidata JSON dump and it's about 90GB, too large to load into memory. It consists of a simple JSON structure like this:
[
item,
item,
item,
...
]
Each "item" looks something like this:
{
"type": "item",
"id": "Q23",
"labels": {
"<lang>": obj
},
"descriptions": {
"<lang>": {
"language": "<lang>",
"value": "<string>"
},
},
"aliases": {
"<key>": [
obj,
obj,
],
},
"claims": {
"<keyID>": [
{
"mainsnak": {
"snaktype": "value",
"property": "<keyID>",
"datavalue": {
"value": {
"entity-type": "<type>",
"numeric-id": <num>,
"id": "<id>"
},
"type": "wikibase-entityid"
},
"datatype": "wikibase-item"
},
"type": "statement",
"id": "<anotherId>",
"rank": "preferred",
"references": [
{
"hash": "<hash>",
"snaks": {
"<keyIDX>": [
{
"snaktype": "value",
"property": "P854",
"datavalue": obj,
"datatype": "url"
}
]
},
"snaks-order": [
"<propID>"
]
}
]
}
]
},
"sitelinks": {
"<lang>wiki": {
"site": "<lang>wiki",
"title": "<string>",
"badges": []
}
}
}
The JSON stream is configured like this:
const fs = require('fs')
const zlib = require('zlib')
const { parser } = require('stream-json')
let stream = fs.createReadStream('./wikidata/latest-all.json.gz')
stream
.pipe(zlib.createGunzip())
.pipe(parser())
.on('data', buildItem)
function buildItem(data) {
switch (data.name) {
case `startArray`:
break
case `startObject`:
break
case `startKey`:
break
case `stringChunk`:
break
case `endKey`:
break
case `keyValue`:
break
case `startString`:
break
case `endString`:
break
case `stringValue`:
break
case `endObject`:
break
case `endArray`:
break
}
}
Notice the buildItem
has the key information, it shows that the JSON stream emits objects like this (these are the logs):
{ name: 'startArray' }
{ name: 'startObject' }
{ name: 'startKey' }
{ name: 'startString' }
{ name: 'stringValue', value: 'type' }
{ name: 'endString' }
...
How do you parse this into item
objects like the above? Parsing this linear stream into a tree is very difficult to comprehend.
A sample of output from the JSON stream is here, which you could use to test a parser if it helps.
from How to parse items from a large JSON stream in JavaScript?
No comments:
Post a Comment