Skip to content

Commit 07ef732

Browse files
r1bKyleAMathews
r1b
authored and
KyleAMathews
committed
gatsby-transformer-{json,yaml}: adds support for flat file structure (#2324)
* gatsby-transformer-yaml: adds support for single objects * gatsby-transformer-json: adds support for single objects * adds realistic node dir * gatsby-transformer-{json,yaml}: updates README.md to describe new data layout
1 parent ce10ef9 commit 07ef732

File tree

8 files changed

+323
-57
lines changed

8 files changed

+323
-57
lines changed

‎packages/gatsby-transformer-json/README.md‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# gatsby-transformer-json
22

3-
Parses JSON files. Currently it only handles stringified arrays but the
4-
goal is to handle in a reasonable way many types of JSON data.
3+
Parses JSON files. Supports arrays of objects and single objects.
54

65
## Install
76

@@ -18,6 +17,11 @@ plugins: [
1817

1918
## Parsing algorithm
2019

20+
You can choose to structure your data as arrays of objects in individual files
21+
or as single objects spread across multiple files.
22+
23+
### Array of Objects
24+
2125
The algorithm for arrays is to convert each item in the array into
2226
a node.
2327

@@ -32,9 +36,59 @@ So if your project has a `letters.json` with `[{ "value": "a" }, { "value":
3236
]
3337
```
3438

39+
### Single Object
40+
41+
The algorithm for single JSON objects is to convert the object defined at the
42+
root of the file into a node. The type of the node is based on the name of the
43+
parent directory.
44+
45+
For example, lets say your project has a data layout like:
46+
47+
```
48+
data/
49+
letters/
50+
a.json
51+
b.json
52+
c.json
53+
```
54+
55+
Where each of `a.json`, `b.json` and `c.json` look like:
56+
57+
```javascript
58+
{ 'value': 'a' }
59+
```
60+
61+
```javascript
62+
{ 'value': 'b' }
63+
```
64+
65+
```javascript
66+
{ 'value': 'c' }
67+
```
68+
69+
Then the following three nodes would be created.
70+
71+
```javascript
72+
[
73+
{
74+
value: 'a',
75+
type: 'Letters',
76+
},
77+
{
78+
value: 'b',
79+
type: 'Letters',
80+
},
81+
{
82+
value: 'c',
83+
type: 'Letters',
84+
},
85+
]
86+
```
87+
3588
## How to query
3689

37-
You'd be able to query your letters like:
90+
Regardless of whether you choose to structure your data in arrays of objects or
91+
single objects, you'd be able to query your letters like:
3892

3993
```graphql
4094
{

‎packages/gatsby-transformer-json/src/__tests__/__snapshots__/gatsby-node.js.snap‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,56 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3+
exports[`Process JSON nodes correctly correctly creates a node from JSON which is a single object 1`] = `
4+
Array [
5+
Array [
6+
Object {
7+
"blue": true,
8+
"children": Array [],
9+
"funny": "yup",
10+
"id": "foo",
11+
"internal": Object {
12+
"contentDigest": "8838e569ae02d98806532310fb2a577a",
13+
"type": "FooJson",
14+
},
15+
"parent": "whatever",
16+
},
17+
],
18+
]
19+
`;
20+
21+
exports[`Process JSON nodes correctly correctly creates a node from JSON which is a single object 2`] = `
22+
Array [
23+
Array [
24+
Object {
25+
"child": Object {
26+
"blue": true,
27+
"children": Array [],
28+
"funny": "yup",
29+
"id": "foo",
30+
"internal": Object {
31+
"contentDigest": "8838e569ae02d98806532310fb2a577a",
32+
"type": "FooJson",
33+
},
34+
"parent": "whatever",
35+
},
36+
"parent": Object {
37+
"children": Array [],
38+
"content": "{\\"id\\":\\"foo\\",\\"blue\\":true,\\"funny\\":\\"yup\\"}",
39+
"dir": "/tmp/foo/",
40+
"id": "whatever",
41+
"internal": Object {
42+
"contentDigest": "whatever",
43+
"mediaType": "application/json",
44+
"name": "test",
45+
},
46+
"name": "nodeName",
47+
"parent": "SOURCE",
48+
},
49+
},
50+
],
51+
]
52+
`;
53+
354
exports[`Process JSON nodes correctly correctly creates nodes from JSON which is an array of objects 1`] = `
455
Array [
556
Array [

‎packages/gatsby-transformer-json/src/__tests__/gatsby-node.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,27 @@ describe(`Process JSON nodes correctly`, () => {
4242
})
4343
})
4444

45+
it(`correctly creates a node from JSON which is a single object`, async () => {
46+
const data = { id: `foo`, blue: true, funny: `yup` }
47+
node.content = JSON.stringify(data)
48+
node.dir = `/tmp/foo/`
49+
50+
const createNode = jest.fn()
51+
const createParentChildLink = jest.fn()
52+
const boundActionCreators = { createNode, createParentChildLink }
53+
54+
await onCreateNode({
55+
node,
56+
loadNodeContent,
57+
boundActionCreators,
58+
}).then(() => {
59+
expect(createNode.mock.calls).toMatchSnapshot()
60+
expect(createParentChildLink.mock.calls).toMatchSnapshot()
61+
expect(createNode).toHaveBeenCalledTimes(1)
62+
expect(createParentChildLink).toHaveBeenCalledTimes(1)
63+
})
64+
})
65+
4566
it(`If the object has an id, it uses that as the id instead of an autogenerated one`, async () => {
4667
const data = [
4768
{ id: `foo`, blue: true, funny: `yup` },

‎packages/gatsby-transformer-json/src/gatsby-node.js‎

Lines changed: 32 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,54 @@
11
const _ = require(`lodash`)
22
const crypto = require(`crypto`)
3+
const path = require(`path`)
34

45
async function onCreateNode({ node, boundActionCreators, loadNodeContent }) {
5-
const { createNode, createParentChildLink } = boundActionCreators
6-
7-
// We only care about JSON content.
8-
if (node.internal.mediaType !== `application/json`) {
9-
return
10-
}
11-
12-
const content = await loadNodeContent(node)
13-
const parsedContent = JSON.parse(content)
14-
15-
// TODO handle non-array data.
16-
if (_.isArray(parsedContent)) {
17-
const JSONArray = parsedContent.map((obj, i) => {
6+
function transformObject(obj, id, type) {
187
const objStr = JSON.stringify(obj)
198
const contentDigest = crypto
209
.createHash(`md5`)
2110
.update(objStr)
2211
.digest(`hex`)
23-
24-
return {
12+
const jsonNode = {
2513
...obj,
26-
id: obj.id ? obj.id : `${node.id} [${i}] >>> JSON`,
14+
id,
2715
children: [],
2816
parent: node.id,
2917
internal: {
3018
contentDigest,
31-
// TODO make choosing the "type" a lot smarter. This assumes
32-
// the parent node is a file.
33-
// PascalCase
34-
type: _.upperFirst(_.camelCase(`${node.name} Json`)),
19+
type,
3520
},
3621
}
37-
})
22+
createNode(jsonNode)
23+
createParentChildLink({ parent: node, child: jsonNode })
24+
}
3825

39-
_.each(JSONArray, j => {
40-
createNode(j)
41-
createParentChildLink({ parent: node, child: j })
42-
})
26+
const { createNode, createParentChildLink } = boundActionCreators
27+
28+
// We only care about JSON content.
29+
if (node.internal.mediaType !== `application/json`) {
30+
return
4331
}
4432

45-
return
33+
const content = await loadNodeContent(node)
34+
const parsedContent = JSON.parse(content)
35+
36+
if (_.isArray(parsedContent)) {
37+
parsedContent.forEach((obj, i) => {
38+
transformObject(
39+
obj,
40+
obj.id ? obj.id : `${node.id} [${i}] >>> JSON`,
41+
_.upperFirst(_.camelCase(`${node.name} Json`)),
42+
)
43+
})
44+
}
45+
else if (_.isPlainObject(parsedContent)) {
46+
transformObject(
47+
parsedContent,
48+
parsedContent.id ? parsedContent.id : `${node.id} >>> JSON`,
49+
_.upperFirst(_.camelCase(`${path.basename(node.dir)} Json`)),
50+
)
51+
}
4652
}
4753

4854
exports.onCreateNode = onCreateNode

‎packages/gatsby-transformer-yaml/README.md‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# gatsby-transformer-yaml
22

3-
Parses YAML files. Currently it only handles YAML arrays but the
4-
goal is to handle in a reasonable way many types of YAML data.
3+
Parses YAML files. Supports arrays of objects and single objects.
54

65
## Install
76

@@ -18,8 +17,13 @@ plugins: [
1817

1918
## Parsing algorithm
2019

20+
You can choose to structure your data as arrays of objects in individual files
21+
or as single objects spread across multiple files.
22+
23+
### Array of Objects
24+
2125
The algorithm for YAML arrays is to convert each item in the array into
22-
a node.
26+
a node. The type of the node is based on the filename.
2327

2428
So if your project has a `letters.yaml` which looks like:
2529

@@ -48,9 +52,59 @@ Then the following three nodes would be created.
4852
]
4953
```
5054

55+
### Single Object
56+
57+
The algorithm for single YAML objects is to convert the object defined at the
58+
root of the file into a node. The type of the node is based on the name of the
59+
parent directory.
60+
61+
For example, lets say your project has a data layout like:
62+
63+
```
64+
data/
65+
letters/
66+
a.yml
67+
b.yml
68+
c.yml
69+
```
70+
71+
Where each of `a.yml`, `b.yml` and `c.yml` look like:
72+
73+
```yaml
74+
value: a
75+
```
76+
77+
```yaml
78+
value: b
79+
```
80+
81+
```yaml
82+
value: c
83+
```
84+
85+
Then the following three nodes would be created.
86+
87+
```javascript
88+
[
89+
{
90+
value: 'a',
91+
type: 'Letters',
92+
},
93+
{
94+
value: 'b',
95+
type: 'Letters',
96+
},
97+
{
98+
value: 'c',
99+
type: 'Letters',
100+
},
101+
]
102+
```
103+
51104
## How to query
52105

53-
You'd be able to query your letters like:
106+
Regardless of whether you choose to structure your data in arrays of objects or
107+
single objects, you'd be able to query your letters like:
54108

55109
```graphql
56110
{

‎packages/gatsby-transformer-yaml/src/__tests__/__snapshots__/gatsby-node.js.snap‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,57 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3+
exports[`Process YAML nodes correctly correctly creates a node from JSON which is a single object 1`] = `
4+
Array [
5+
Array [
6+
Object {
7+
"blue": true,
8+
"children": Array [],
9+
"funny": "yup",
10+
"id": "whatever >>> YAML",
11+
"internal": Object {
12+
"contentDigest": "73901821b17d5aa9dd6026181f73b64c",
13+
"type": "BarYaml",
14+
},
15+
"parent": "whatever",
16+
},
17+
],
18+
]
19+
`;
20+
21+
exports[`Process YAML nodes correctly correctly creates a node from JSON which is a single object 2`] = `
22+
Array [
23+
Array [
24+
Object {
25+
"child": Object {
26+
"blue": true,
27+
"children": Array [],
28+
"funny": "yup",
29+
"id": "whatever >>> YAML",
30+
"internal": Object {
31+
"contentDigest": "73901821b17d5aa9dd6026181f73b64c",
32+
"type": "BarYaml",
33+
},
34+
"parent": "whatever",
35+
},
36+
"parent": Object {
37+
"children": Array [],
38+
"content": "blue: true
39+
funny: yup
40+
",
41+
"dir": "/tmp/bar/",
42+
"id": "whatever",
43+
"internal": Object {
44+
"contentDigest": "whatever",
45+
"mediaType": "text/yaml",
46+
},
47+
"name": "test",
48+
"parent": "SOURCE",
49+
},
50+
},
51+
],
52+
]
53+
`;
54+
355
exports[`Process YAML nodes correctly correctly creates nodes from JSON which is an array of objects 1`] = `
456
Array [
557
Array [

0 commit comments

Comments
 (0)