From c83b85ab531f4f17aa3624a2f144e03960d2e1d4 Mon Sep 17 00:00:00 2001 From: Kyle Gill Date: Thu, 9 Apr 2020 10:00:58 -0600 Subject: [PATCH] docs: update creating source plugins guide & add example repo (#22943) * add example monorepo, make initial updates for source plugin guide and move some content to the transformer guide * update sample code in guide for proactive fetching * update READMEs and remove unused files * remove more unused files * Apply suggestions from code review Co-Authored-By: LB * split terminal comments up * Apply suggestions from code review Co-Authored-By: LB * code suggestions * more review suggestions Co-authored-by: LB --- docs/docs/creating-a-source-plugin.md | 488 +++++++++++++++--- docs/docs/creating-a-transformer-plugin.md | 52 +- examples/creating-source-plugins/README.md | 83 +++ .../creating-source-plugins/api/README.md | 3 + .../creating-source-plugins/api/package.json | 43 ++ .../creating-source-plugins/api/src/index.js | 138 +++++ .../api/src/schema.graphql | 30 ++ .../example-site/README.md | 3 + .../example-site/gatsby-config.js | 22 + .../example-site/package.json | 34 ++ .../example-site/src/pages/index.js | 72 +++ .../example-site/static/favicon.ico | Bin 0 -> 2813 bytes examples/creating-source-plugins/package.json | 14 + .../source-plugin/README.md | 3 + .../source-plugin/gatsby-node.js | 274 ++++++++++ .../source-plugin/index.js | 1 + .../source-plugin/package.json | 28 + 17 files changed, 1194 insertions(+), 94 deletions(-) create mode 100644 examples/creating-source-plugins/README.md create mode 100644 examples/creating-source-plugins/api/README.md create mode 100644 examples/creating-source-plugins/api/package.json create mode 100644 examples/creating-source-plugins/api/src/index.js create mode 100644 examples/creating-source-plugins/api/src/schema.graphql create mode 100644 examples/creating-source-plugins/example-site/README.md create mode 100644 examples/creating-source-plugins/example-site/gatsby-config.js create mode 100644 examples/creating-source-plugins/example-site/package.json create mode 100644 examples/creating-source-plugins/example-site/src/pages/index.js create mode 100644 examples/creating-source-plugins/example-site/static/favicon.ico create mode 100644 examples/creating-source-plugins/package.json create mode 100644 examples/creating-source-plugins/source-plugin/README.md create mode 100644 examples/creating-source-plugins/source-plugin/gatsby-node.js create mode 100644 examples/creating-source-plugins/source-plugin/index.js create mode 100644 examples/creating-source-plugins/source-plugin/package.json diff --git a/docs/docs/creating-a-source-plugin.md b/docs/docs/creating-a-source-plugin.md index 0af8ac02d0e47..fa905bcb9711b 100644 --- a/docs/docs/creating-a-source-plugin.md +++ b/docs/docs/creating-a-source-plugin.md @@ -4,11 +4,11 @@ title: Creating a Source Plugin Source plugins are essentially out of the box integrations between Gatsby and various third-party systems. -These systems can be CMSs like Contentful or WordPress, other cloud services like Lever and Strava, or your local filesystem -- literally anything that has an API. Currently, Gatsby has [over 300 source plugins](/plugins/?=gatsby-source). +These systems can be CMSs like Contentful or WordPress, other cloud services like Lever and Strava, or your local filesystem -- literally anything that has an API. Currently, Gatsby has [over 400 source plugins](/plugins/?=gatsby-source). Once a source plugin brings data into Gatsby's system, it can be transformed further with **transformer plugins**. For step-by-step examples of how to create source and transformer plugins, check out the Gatsby [tutorials section](/tutorial/plugin-and-theme-tutorials/). -## What do source plugins do? +## Overview of a source plugin At a high-level, a source plugin: @@ -18,26 +18,59 @@ At a high-level, a source plugin: - Links nodes & creates relationships between them. - Lets Gatsby know when nodes are finished sourcing so it can move on to processing them. -## What does the code look like? +A source plugin is a regular npm package. It has a `package.json` file, with optional dependencies, as well as a [`gatsby-node.js`](/docs/api-files-gatsby-node) file where you implement Gatsby's [Node APIs](/docs/node-apis/). Read more about [files Gatsby looks for in a plugin](/docs/files-gatsby-looks-for-in-a-plugin/) or [creating a generic plugin](/docs/creating-a-generic-plugin). -A source plugin is a regular NPM package. It has a `package.json` file with optional -dependencies as well as a [`gatsby-node.js`](/docs/api-files-gatsby-node) file where you implement Gatsby's [Node -APIs](/docs/node-apis/). Read more about [Files Gatsby Looks for in a Plugin](/docs/files-gatsby-looks-for-in-a-plugin/). +## Implementing features for source plugins -Gatsby's minimum supported Node.js version is Node 8 and as it's common to want to use more modern Node.js and JavaScript syntax, many plugins write code in a -source directory and compile the code. All plugins maintained in the Gatsby repo -follow this pattern. +Key features that are often built into source plugins are covered in this guide to help explain Gatsby specific helpers and APIs, independent of the source the data is coming from. -Your `gatsby-node.js` should look something like: +> You can see examples of all the features implemented in this guide (sourcing data, caching, live data synchronization, and remote image optimization) **in the working example repository** for [creating source plugins](https://github.com/gatsbyjs/gatsby/tree/master/examples/creating-source-plugins) which contains a local server you can run to test with an example source plugin. -```javascript:title=gatsby-node.js +### Sourcing data and creating nodes + +All source plugins must fetch data and create nodes from that data. By fetching data and creating nodes at [build time](/docs/glossary#build), Gatsby can make the data available as static assets instead of having to fetch it at [runtime](/docs/glossary#runtime). This happens in the [`sourceNodes` lifecycle](/docs/node-apis/#sourceNodes) with the [`createNode` action](/docs/actions/#createNode). + +This example—taken from [the `sourceNodes` API docs](/docs/node-apis/#sourceNodes)—shows how to create a single node from hardcoded data: + +```javascript:title=source-plugin/gatsby-node.js +exports.sourceNodes = ({ actions, createNodeId, createContentDigest }) => { + const { createNode } = actions + + // Data can come from anywhere, but for now create it manually + const myData = { + key: 123, + foo: `The foo field of my node`, + bar: `Baz`, + } + + const nodeContent = JSON.stringify(myData) + + const nodeMeta = { + id: createNodeId(`my-data-${myData.key}`), + parent: null, + children: [], + internal: { + type: `MyNodeType`, + mediaType: `text/html`, + content: nodeContent, + contentDigest: createContentDigest(myData), + }, + } + + const node = Object.assign({}, myData, nodeMeta) + createNode(node) +} +``` + +Source plugins follow the same pattern, the only difference is that data comes from other sources. Plugins can leverage Node.js built-in functions like `http.get`, libraries like `node-fetch` or `axios`, or even fully-featured GraphQL clients to fetch data. With data being returned from a remote location, the plugin code can loop through and create nodes programmatically: + +```javascript:title=source-plugin/gatsby-node.js exports.sourceNodes = async ({ actions }) => { const { createNode } = actions - // Create nodes here, generally by downloading data - // from a remote API. + // Download data from a remote API. const data = await fetch(REMOTE_API) - // Process data into nodes. + // Process data and create nodes.using a custom processDatum function data.forEach(datum => createNode(processDatum(datum))) // You're done, return. @@ -45,120 +78,409 @@ exports.sourceNodes = async ({ actions }) => { } ``` -Peruse the [`sourceNodes`](/docs/node-apis/#sourceNodes) and -[`createNode`](/docs/actions/#createNode) docs for detailed -documentation on implementing those APIs. +The [`createNode`](/docs/actions/#createNode) function is a Gatsby specific action. `createNode` is used to create the nodes that Gatsby tracks and makes available for querying with GraphQL. -### Transforming data received from remote sources +_Note: **Be aware of asynchronous operations!** Because fetching data is an asynchronous task, you need to make sure you `await` data coming from remote sources, return a Promise, or return the callback (the 3rd parameter available in lifecycle APIs) from `sourceNodes`. If you don't, Gatsby will continue on in the build process, before nodes are finished being created. This can result in your nodes not ending up in the generated schema at compilation time, or the process could hang while waiting for an indication that it's finished. You can read more in the [Debugging Asynchronous Lifecycle APIs guide](/docs/debugging-async-lifecycles/)._ -Each node created by the filesystem source plugin includes the -raw content of the file and its _media type_. +### Caching data between runs -[A **media type**](https://en.wikipedia.org/wiki/Media_type) (also **MIME type** -and **content type**) is an official way to identify the format of -files/content that is transmitted on the internet, e.g. over HTTP or through -email. You might be familiar with other media types such as -`application/javascript`, `application/pdf`, `audio/mpeg`, `text/html`, -`text/plain`, `image/jpeg`, etc. +Some operations like fetching data from an endpoint can be performance heavy or time-intensive. In order to improve the experience of developing with your source plugin, you can leverage the Gatsby cache to store data between runs of `gatsby develop` or `gatsby build`. -Each source plugin is responsible for setting the media type for the nodes they -create. This way, source and transformer plugins can work together easily. +You access the `cache` in Gatsby Node APIs and use the `set` and `get` functions to store and retrieve data as JSON objects. -This is not a required field -- if it's not provided, Gatsby will [infer](/docs/glossary#inference) the type from data that is sent -- but it's the way for source plugins to indicate to -transformers that there is "raw" data that can still be further processed. It -also allows plugins to remain small and focused. Source plugins don't have to have -opinions on how to transform their data: they can set the `mediaType` and -push that responsibility to transformer plugins, instead. +```javascript:title=source-plugin/gatsby-node.js +exports.onPostBuild = async ({ cache }) => { + await cache.set(`key`, `value`) + const cachedValue = await cache.get(`key`) + console.log(cachedValue) // logs `value` +} +``` -For example, it's common for services to allow you to add content in -Markdown format. If you pull that Markdown into Gatsby and create a new node, what -then? How would a user of your source plugin convert that Markdown into HTML -they can use in their site? You would create a -node for the Markdown content and set its `mediaType` as `text/markdown` and the -various Gatsby Markdown transformer plugins would see your node and transform it -into HTML. +The above snippet shows a contrived example for the `cache`, but it can be used in more sophisticated cases to reduce the time it takes to run your plugin. For example, by caching a timestamp, you can use it to fetch solely the data that has been updated since the last time data was fetched from the source: -This loose coupling between the data source and the transformer plugins allow Gatsby site builders to assemble complex data transformation pipelines with -little work on their (and your (the source plugin author)) part. +```javascript:title=source-plugin/gatsby-node.js +exports.sourceNodes = async ({ cache }) => { + // get the last timestamp from the cache + const lastFetched = await cache.get(`timestamp`) -## Getting helper functions + // pull data from some remote source using cached data as an option in the request + const data = await fetch( + `https://remotedatasource.com/posts?lastUpdated=${lastFetched}` + ) + // ... +} -[`gatsby-node-helpers`](https://github.com/angeloashmore/gatsby-node-helpers), -a community-made NPM package, can help when writing source plugins. This -package provides a set of helper functions to generate Node objects with the -required fields. This includes automatically generating fields like node IDs -and the `contentDigest` MD5 hash, keeping your code focused on data gathering, -not boilerplate. +exports.onPostBuild = async ({ cache }) => { + // set a timestamp at the end of the build + await cache.set(`timestamp`, Date.now()) +} +``` -## Gotcha: don't forget to return! +> In addition to the cache, plugins can save metadata to the [internal Redux store](/docs/data-storage-redux/) with `setPluginStatus`. -After your plugin is finished sourcing nodes, it should either return a Promise or use the callback (3rd parameter) to report back to Gatsby when `sourceNodes` is fully executed. If a Promise or callback isn't returned, Gatsby will continue on in the build process, before nodes are finished being created. Without the necessary return statement your nodes might not end up in the generated schema at compilation time, or the process will hang while waiting for an indication that it's finished. +This can reduce the time it takes repeated data fetching operations to run if you are pulling in large amounts of data for your plugin. Existing plugins like [`gatsby-source-contentful`](https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-source-contentful/src/gatsby-node.js) generate a token that is sent with each request to only return new data. -## Advanced +You can read more about the cache API, other types of plugins that leverage the cache, and example open source plugins that use the cache in the [build caching guide](/docs/build-caching). ### Adding relationships between nodes Gatsby source plugins not only create nodes, they also create relationships between nodes that are exposed to GraphQL queries. -There are two ways of adding node relationships in Gatsby: (1) transformations (parent-child) or (2) foreign-key based. - -#### Option 1: transformation relationships +There are two types of node relationships in Gatsby: (1) foreign-key based and (2) transformations (parent-child). -An example of a transformation relationship is the `gatsby-transformer-remark` plugin, which transforms a parent `File` node's markdown string into a `MarkdownRemark` node. The Remark transformer plugin adds its newly created child node as a child of the parent node using the action [`createParentChildLink`](/docs/actions/#createParentChildLink). Transformation relationships are used when a new node is _completely_ derived from a single parent node. E.g. the markdown node is derived from the parent `File` node and wouldn't ever exist if the parent `File` node hadn't been created. +#### Option 1: foreign-key relationships -Because all children nodes are derived from their parent, when a parent node is deleted or changed, Gatsby deletes all of the child nodes (and their child nodes, and so on) with the expectation that they'll be recreated again by transformer plugins. This is done to ensure there are no nodes left over that were derived from older versions of data but shouldn't exist any longer. +An example of a foreign-key relationship would be a `Post` type (like a blog post) that has an `Author`. -_Creating the transformation relationship_ +In this relationship, each object is a distinct entity that exists whether or not the other does. They could each be queried individually. -In order to create a parent/child relationship, when calling `createNode` for the child node, the new node object that is passed in should have a `parent` key with the value set to the parent node's `id`. After this, call the `createParentChildLink` function exported inside `actions`. - -_Examples_ +```graphql +post { + id + title +} +author { + id + name +} +``` -[Here's the above example](https://github.com/gatsbyjs/gatsby/blob/72077527b4acd3f2109ed5a2fcb780cddefee35a/packages/gatsby-transformer-remark/src/on-node-create.js#L39-L67) from the `gatsby-transformer-remark` source plugin. +Each type has independent schemas and field(s) on that reference the other entity -- in this case the `Post` would have an `Author`, and the `Author` might have `Post`s. The API of a service that allows complex object modelling, for example a CMS, will often allow users to add relationships between entities and expose them through the API. This same relationship can be represented by your schema. + +```graphql +post { + id + title + // highlight-start + author { + id + name + } + // highlight-end +} +author { + id + name + // highlight-start + posts { + id + title + } + // highlight-end +} +``` -[Here's another example](https://github.com/gatsbyjs/gatsby/blob/1fb19f9ad16618acdac7eda33d295d8ceba7f393/packages/gatsby-transformer-sharp/src/on-node-create.js#L3-L25) from the `gatsby-transformer-sharp` source plugin. +When an object node is deleted, Gatsby _does not_ delete any referenced entities. When using foreign-key references, it's a source plugin's responsibility to clean up any dangling entity references. -#### Option 2: foreign-key relationships +##### Creating the relationship -An example of a foreign-key relationship would be a Post that has an Author. +Suppose you want to create a relationship between `Post`s and `Author`s in order to query the `author` field on a post: + +```graphql +query { + post { + id + // highlight-start + author { + id + name + } + // highlight-end + } +} +``` -In this relationship, each object is a distinct entity that exists whether or not the other does, with independent schemas, and field(s) on each entity that reference the other entity -- in this case the Post would have an Author, and the Author might have Posts. The API of a service that allows complex object modelling, for example a CMS, will often allow users to add relationships between entities and expose them through the API. +For Gatsby to automatically infer a relationship, you need to create a field called `author___NODE` on the Post object to hold the relationship to Authors before you create the node. The value of this field should be the node ID of the Author. -When an object node is deleted, Gatsby _does not_ delete any referenced entities. When using foreign-key references, it's a source plugin's responsibility to clean up any dangling entity references. +```javascript:title=source-plugin/gatsby-node.js +exports.sourceNodes = ({ actions, createContentDigest }) => { + const { createNode } = actions + createNode({ + // Data for the Post node + author___NODE: ``, // highlight-line + // Required fields + id: `a-node-id`, + parent: null + children: [], + internal: { + type: `post`, + contentDigest: createContentDigest(fieldData), + } + }) +} +``` -##### Creating the relationship +For a stricter GraphQL schema, you can specify the exact field and value to link nodes using schema customization APIs. -Suppose you want to create a relationship between Posts and Authors, and you want to call the field `author`. +```javascript:title=source-plugin/gatsby-node.js +exports.sourceNodes = ({ actions, createContentDigest }) => { + const { createNode } = actions + createNode({ + // Data for the Post node + // highlight-start + author: { + name: `Jay Gatsby`, + }, + // highlight-end + // Required fields + id: `a-node-id`, + parent: null + children: [], + internal: { + type: `post`, + contentDigest: createContentDigest(fieldData), + } + }) +} -Before you pass the Post object and Author object into `createNode` and create the respective nodes, you need to create a field called `author___NODE` on the Post object to hold the relationship to Authors. The value of this field should be the node ID of the Author. +exports.createSchemaCustomization = ({ actions }) => { + const { createTypes } = actions + createTypes(` + type Post implements Node { + id: ID! + # create a relationship between Post and the File nodes for optimized images + author: Author @link(from: "author.name" by: "name") // highlight-line + # ... other fields + }`) +} +``` ##### Creating the reverse relationship -It's often convenient for querying to add to the schema backwards references. For example, you might want to query the Author of a Post but you might also want to query all the posts an author has written. +It's often convenient for querying to add to the schema backwards references. For example, you might want to query the author of a post, but you might also want to query all the posts an author has written. -If you want to call this field on `Author` `posts`, you would create a field called `posts___NODE` to hold the relationship to Posts. The value of this field should be an array of Post IDs. +If you want to call a field to access the author on the `Post` nodes using the inference method, you would create a field called `posts___NODE` to hold the relationship to posts. The value of this field should be an array of `Post` IDs. Here's an example from the [WordPress source plugin](https://github.com/gatsbyjs/gatsby/blob/1fb19f9ad16618acdac7eda33d295d8ceba7f393/packages/gatsby-source-wordpress/src/normalize.js#L178-L189). +With schema customization, you would add the `@link` directive to your Author type. The `@link` directive will look for an ID on the `post` field of the Author nodes, which can be added when the Author nodes are created. + +```javascript:title=source-plugin/gatsby-node.js +exports.createSchemaCustomization = ({ actions }) => { + const { createTypes } = actions + createTypes(` + type Post implements Node { + id: ID! + # create a relationship between Post and the File nodes for optimized images + author: Author @link(from: "author.name" by: "name") // highlight-line + # ... other fields + } + + type Author implements Node { + name: String! + post: Post @link // highlight-line + }`) +} +``` + +You can read more about connecting foreign key fields with schema customization in the guide on [customizing the GraphQL schema](/docs/schema-customization/#foreign-key-fields). + +#### Option 2: transformation relationships + +When a node is _completely_ derived from another node you'll want to use a transformation relationship. An example that is common in source plugins is for transforming File nodes from remote sources, e.g. images. You can read about this use case in the section below on [sourcing images from remote locations](/docs/creating-a-source-plugin/#sourcing-images-from-remote-locations). + +You can find more information about transformation relationships in the [creating a transformer plugin guide](/docs/creating-a-transformer-plugin/#creating-the-transformer-relationship). + #### Union types -When creating fields linking to an array of nodes, if the array of IDs are all of the same type, the relationship field that is created will be of this type. If the linked nodes are of different types; the field will turn into a union type of all types that are linked. See the [GraphQL documentation on how to query union types](https://graphql.org/learn/schema/#union-types). +For either type of relationship you can link a field to an array of nodes. If the array of IDs all correspond to nodes of the same type, the relationship field that is created will be of this type. If the linked nodes are of different types the field will turn into a union type of all types that are linked. See the [GraphQL documentation on how to query union types](https://graphql.org/learn/schema/#union-types). + +### Working with data received from remote sources + +#### Setting media and MIME types + +Each node created by the filesystem source plugin includes the raw content of the file and its _media type_. + +[A **media type**](https://en.wikipedia.org/wiki/Media_type) (also **MIME type** and **content type**) is an official way to identify the format of files/content that are transmitted via the internet, e.g. over HTTP or through email. You might be familiar with other media types such as `application/javascript`, `audio/mpeg`, `text/html`, etc. + +Each source plugin is responsible for setting the media type for the nodes it creates. This way, source and transformer plugins can work together easily. + +This is not a required field -- if it's not provided, Gatsby will [infer](/docs/glossary#inference) the type from data that is sent -- but it's how source plugins indicate to transformers that there is "raw" data the transformer can further process. + +It also allows plugins to remain small and focused. Source plugins don't have to have opinions on how to transform their data: they can set the `mediaType` and push that responsibility to transformer plugins instead. + +For example, it's common for services to allow you to add content in Markdown format. If you pull that Markdown into Gatsby and create a new node, what then? How would a user of your source plugin convert that Markdown into HTML they can use in their site? You would create a node for the Markdown content and set its `mediaType` as `text/markdown` and the various Gatsby Markdown transformer plugins would see your node and transform it into HTML. -#### Further specification +This loose coupling between the data source and the transformer plugins allow Gatsby site builders to assemble complex data transformation pipelines with little work on their (and your (the source plugin author)) part. -See -[_Node Link_](/docs/api-specification/) in the API Specification concepts -section for more info. +#### Sourcing and optimizing images from remote locations + +A common use case for source plugins is pulling images from a remote location and optimizing them for use with [Gatsby Image](/packages/gatsby-image/). An API may return a URL for an image on a CDN, which could be further optimized by Gatsby at build time. + +This can be achieved by the following steps: + +1. Install `gatsby-source-filesystem` as a dependency in your source plugin: + +``` +npm install gatsby-source-filesystem +``` + +2. Create File nodes using the `createRemoteFileNode` function exported by `gatsby-source-filesystem`: + +```javascript:title=source-plugin/gatsby-node.js +const { createRemoteFileNode } = require(`gatsby-source-filesystem`) + +exports.onCreateNode = async ({ + actions: { createNode }, + getCache, + createNodeId, + node, +}) => { + // because onCreateNode is called for all nodes, verify that you are only running this code on nodes created by your plugin + if (node.internal.type === `your-source-node-type`) { + // create a FileNode in Gatsby that gatsby-transformer-sharp will create optimized images for + const fileNode = await createRemoteFileNode({ + // the url of the remote image to generate a node for + url: node.imgUrl, + getCache, + createNode, + createNodeId, + parentNodeId: node.id, + }) + } +} +``` + +3. Add the ID of the new File node to your source plugin's node. + +```javascript:title=source-plugin/gatsby-node.js +const { createRemoteFileNode } = require(`gatsby-source-filesystem`) + +exports.onCreateNode = async ({ + actions: { createNode }, + getCache, + createNodeId, + node, +}) => { + // because onCreateNode is called for all nodes, verify that you are only running this code on nodes created by your plugin + if (node.internal.type === `your-source-node-type`) { + // create a FileNode in Gatsby that gatsby-transformer-sharp will create optimized images for + const fileNode = await createRemoteFileNode({ + // the url of the remote image to generate a node for + url: node.imgUrl, + getCache, + createNode, + createNodeId, + parentNodeId: node.id, + }) + + // highlight-start + if (fileNode) { + // with schemaCustomization: add a field `remoteImage` to your source plugin's node from the File node + node.remoteImage = fileNode.id + + // OR with inference: link your source plugin's node to the File node without schemaCustomization like this, but creates a less sturdy schema + node.remoteImage___NODE = fileNode.id + } + // highlight-end + } +} +``` + +Attaching `fileNode.id` to `remoteImage___NODE` will rely on Gatsby's [inference](/docs/glossary/#inference) of the GraphQL schema to create a new field `remoteImage` as a relationship between the nodes. This is done automatically. For a sturdier schema, you can relate them using [`schemaCustomization` APIs](/docs/node-apis/#createSchemaCustomization) by adding the `fileNode.id` to a field that you reference when you `createTypes`: + +```javascript:title=source-plugin/gatsby-node.js +exports.createSchemaCustomization = ({ actions }) => { + const { createTypes } = actions + createTypes(` + type YourSourceType implements Node { + id: ID! + # create a relationship between YourSourceType and the File nodes for optimized images + remoteImage: File @link // highlight-line + }`) +} +``` + +4. Verify that `gatsby-plugin-sharp` and `gatsby-transformer-sharp` are included in the site that is using the plugin: + +```javascript:title=gatsby-config.js +module.exports = { + plugins: [ + // loads the source-plugin + `your-source-plugin`, + // required to generate optimized images + `gatsby-plugin-sharp`, + `gatsby-transformer-sharp`, + ], +} +``` + +Then, the sharp plugins will automatically transform the File nodes created by `createRemoteFileNode` in `your-source-plugin` (which have supported image extensions like .jpg or .png). You can then query for the `remoteImage` field on your source type: + +```graphql +query { + yourSourceType { + id + remoteImage { + childImageSharp { + # fluid or fixed fields for optimzed images + } + } + } +} +``` ### Improve plugin developer experience by enabling faster sync -One tip to improve the development experience of using a plugin is to reduce the time it takes to sync between Gatsby and the data source. There are two approaches for doing this: +One challenge when developing locally is that a developer might make modifications in a remote data source, like a CMS, and then want to see how it looks in the local environment. Typically they will have to restart the `gatsby develop` server to see changes. In order to improve the development experience of using a plugin, you can reduce the time it takes to sync between Gatsby and the data source by enabling faster synchronization of data changes. There are two approaches for doing this: + +- **Proactively fetch updates**. You can avoid having to restart the `gatsby develop` server by proactively fetching updates from the remote server. For example, [gatsby-source-sanity](https://github.com/sanity-io/gatsby-source-sanity) listens to changes to Sanity content when `watchMode` is enabled and pulls them into the Gatsby develop server. The [example source plugin repository](https://github.com/gatsbyjs/gatsby/tree/master/examples/creating-source-plugins) uses GraphQL subscriptions to listen for changes and update data. +- **Add event-based sync**. Some data sources keep event logs and are able to return a list of objects modified since a given time. If you're building a source plugin, you can store the last time you fetched data using the [cache](/docs/creating-a-source-plugin/#caching-data-between-runs) or [`setPluginStatus`](/docs/actions/#setPluginStatus) and then only sync down nodes that have been modified since that time. [gatsby-source-contentful](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful) is an example of a source plugin that does this. + +If possible, the proactive listener approach creates the best experience if existing APIs in the data source can support it (or you have access to build support into the data source). + +The code to support this behavior looks like this: + +```javascript:title=source-plugin/gatsby-node.js +exports.sourceNodes = async ({ actions }, pluginOptions) => { + const { createNode, touchNode, deleteNode } = actions + + // highlight-start + // touch nodes to ensure they aren't garbage collected + getNodesByType(`YourSourceType`).forEach(node => touchNode({ nodeId: node.id })) + + // ensure a plugin is in a preview mode and/or supports listening + if (pluginOptions.preview) { + const subscription = await subscription(SUBSCRIPTION_TO_WEBSOCKET) + subscription.subscribe(({ data: newData }) => { + newData.forEach(newDatum => { + switch (newDatum.status) { + case "deleted": + deleteNode({ + node: getNode(createNodeId(`YourSourceType-${newDatum.uuid}`)), + }) + break + case "created": + case "updated": + default: + // created and updated can be handled by the same code path + // the post's id is presumed to stay constant (or can be inferred) + createNode(processDatum(newDatum)) + break + ) + } + }) + } + // highlight-end + + const data = await client.query(QUERY_TO_API) + + // Process data and create nodes.using a custom processDatum function + data.forEach(datum => createNode(processDatum(datum))) + + // You're done, return. + return +} +``` + +_Note: This is pseudo code to illustrate the logic and concept of how these plugins function, you can see an example in the [creating source plugins](https://github.com/gatsbyjs/gatsby/tree/master/examples/creating-source-plugins) repository._ + +Because the code in `sourceNodes` is reinvoked when changes in the data source occur, a few steps need to be taken to ensure that Gatsby is tracking the existing nodes as well as the new data. A first step is ensuring that the existing nodes created are not garbage collected which is done by "touching" the nodes with the [`touchNode` action](/docs/actions/#touchNode). -- **Add event-based sync**. Some data sources keep event logs and are able to return a list of objects modified since a given time. If you're building a source plugin, you can store - the last time you fetched data using - [`setPluginStatus`](/docs/actions/#setPluginStatus) and then only sync down nodes that have been modified since that time. [gatsby-source-contentful](https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful) is an example of a source plugin that does this. -- **Proactively fetch updates**. One challenge when developing locally is that a developer might make modifications in a remote data source, like a CMS, and then want to see how it looks in the local environment. Typically they will have to restart the `gatsby develop` server to see changes. This can be avoided if your source plugin knows to proactively fetch updates from the remote server. For example, [gatsby-source-sanity](https://github.com/sanity-io/gatsby-source-sanity), listens to changes to Sanity content when `watchMode` is enabled and pulls them into the Gatsby develop server. +Then the new data needs to be pulled in via a live update like a websocket (in the example above with a subscription). The new data needs to have some information attached that dictates whether the data was created, updated, or deleted; that way, when it is processed, a new node can be created/updated (with `createNode`) or deleted (with `deleteNode`). In the example above that information is coming from `newDatum.status`. ## Additional resources -- Tutorial: [Creating a Pixabay Image Source Plugin](/tutorial/pixabay-source-plugin-tutorial/) +- Working example repository on [creating source plugins](https://github.com/gatsbyjs/gatsby/tree/master/examples/creating-source-plugins) with the features in this guide implemented +- Tutorial on [Creating a Pixabay Image Source Plugin](/tutorial/pixabay-source-plugin-tutorial/) +- [`gatsby-node-helpers`](https://github.com/angeloashmore/gatsby-node-helpers), a community-made npm package with helper functions to generate Node objects with required fields like IDs and the `contentDigest` MD5 hash. diff --git a/docs/docs/creating-a-transformer-plugin.md b/docs/docs/creating-a-transformer-plugin.md index 5ec01d2a7945a..19358dc4989eb 100644 --- a/docs/docs/creating-a-transformer-plugin.md +++ b/docs/docs/creating-a-transformer-plugin.md @@ -102,6 +102,8 @@ Now you have a `File` node to work with: Now, transform the newly created `File` nodes by hooking into the `onCreateNode` API in `gatsby-node.js`. +#### Convert yaml into JSON for storage in Gatsby nodes + If you're following along in an example project, install the following packages: ```shell @@ -137,41 +139,67 @@ File content: Parsed YAML content: -```javascript -;[ +```json +[ { - id: "Jane Doe", - bio: "Developer based in Somewhere, USA", + "id": "Jane Doe", + "bio": "Developer based in Somewhere, USA" }, { - id: "John Smith", - bio: "Developer based in Maintown, USA", - }, + "id": "John Smith", + "bio": "Developer based in Maintown, USA" + } ] ``` Now you'll write a helper function to transform the parsed YAML content into new Gatsby nodes: -```javascript +```javascript:title=gatsby-node.js function transformObject(obj, id, type) { const yamlNode = { ...obj, id, children: [], - parent: node.id, + parent: null, internal: { contentDigest: createContentDigest(obj), type, }, } createNode(yamlNode) - createParentChildLink({ parent: node, child: yamlNode }) } ``` Above, you create a `yamlNode` object with the shape expected by the [`createNode` action](/docs/actions/#createNode). -You then create a link between the parent node (file) and the child node (yaml content). +#### Creating the transformer relationship + +You then need to create a link between the parent node (file) and the child node (yaml content) using the `createParentChildLink` function after adding the parent node's id to the `yamlNode`: + +```javascript:title=gatsby-node.js +function transformObject(obj, id, type) { + const yamlNode = { + ...obj, + id, + children: [], + parent: node.id, // highlight-line + internal: { + contentDigest: createContentDigest(obj), + type, + }, + } + createNode(yamlNode) + createParentChildLink({ parent: node, child: yamlNode }) // highlight-line +} +``` + +Another example of a transformation relationship is the `gatsby-source-filesystem` plugin used with the `gatsby-transformer-remark` plugin. This combination transforms a parent `File` node's markdown string into a `MarkdownRemark` node. The remark transformer plugin adds its newly created child node as a child of the parent node using the action [`createParentChildLink`](/docs/actions/#createParentChildLink). Transformation relationships like this are used when a new node is _completely_ derived from a single parent node. E.g. the markdown node is derived from the parent `File` node and would not exist if the parent `File` node hadn't been created. + +Because all children nodes are derived from their parent, when a parent node is deleted or changed, Gatsby deletes all of the child nodes (and their child nodes, and so on). Gatsby does so with the expectation that they'll be recreated again by transformer plugins. This is done to ensure there are no nodes left over that were derived from older versions of data but should no longer exist. + +_For examples of other plugins creating transformation relationships, you can see the [`gatsby-transformer-remark` plugin](https://github.com/gatsbyjs/gatsby/blob/72077527b4acd3f2109ed5a2fcb780cddefee35a/packages/gatsby-transformer-remark/src/on-node-create.js#L39-L67) (from the above example) or the [`gatsby-transformer-sharp` plugin](https://github.com/gatsbyjs/gatsby/blob/1fb19f9ad16618acdac7eda33d295d8ceba7f393/packages/gatsby-transformer-sharp/src/on-node-create.js#L3-L25)._ + +#### Create new nodes from the derived data In your updated `gatsby-node.js`, you'll then iterate through the parsed YAML content, using the helper function to transform each into a new node: @@ -227,6 +255,8 @@ async function onCreateNode({ exports.onCreateNode = onCreateNode ``` +#### Query for the transformed data + Now you can query for your new nodes containing our transformed YAML data: ```graphql diff --git a/examples/creating-source-plugins/README.md b/examples/creating-source-plugins/README.md new file mode 100644 index 0000000000000..fb1face66c326 --- /dev/null +++ b/examples/creating-source-plugins/README.md @@ -0,0 +1,83 @@ +# Creating First Class Gatsby Source Plugins + +Create Gatsby plugins that leverage Gatsby's most impactful native features like remote image optimization, caching, customized GraphQL schemas and node relationships, and more. + +This monorepo serves as an example of a site using a first class source plugin to pull in data from a Node.js API. It is meant to show the 3 pieces that work together when building a source plugin: the API, the site, and the source plugin. + +## Setup + +This monorepo uses yarn workspaces to manage the 3 indivdual projects: + +- api: a Node.js API with in-memory data, and a Post and Author type, as well as support for subscriptions when Posts are mutated +- example-site: a barebones Gatsby site that implements the source plugin +- source-plugin: a plugin that uses several Gatsby APIs to source data from the API, create responsive/optimized images from remote locations, and link the nodes in the example site + +To install dependencies for all projects run the install command in the root of the yarn workspace (which requires yarn to be installed): + +``` +yarn install +``` + +_Note: if you aren't using yarn, you can navigate into each of the 3 folders and run `npm install` instead_ + +Then you can run the api or example projects in separate terminal windows with the commands below. + +For the API which runs at `localhost:4000`, use this command: + +``` +yarn workspace api start +``` + +And to run the example site with `gatsby develop` at `localhost:8000`, use this command: + +``` +yarn workspace example-site develop +``` + +Running the example site also runs the plugin because it is included in the site's config. You'll see output in the console for different functionality and then can open up the browser to `localhost:8000` to see the site. + +## Developing and Experimenting + +You can open up `localhost:4000` with the API running, which will load a GraphQL Playground, which is a GraphQL IDE (like GraphiQL, that Gatsby runs at `localhost:8000/___graphql`) for running queries and mutations on the data from the API. + +You can test a query like this to see data returned: + +```graphql +query { + posts { + id + slug + } +} +``` + +This query will return the IDs for all posts in the API. You can copy one of these IDs and provide it as an argument to a mutation to update information about that post. + +You can run 3 different mutations from the GraphQL Playground (at `localhost:4000`): `createPost`, `updatePost`, and `deletePost`. These methods would mimic CRUD operations happening on the API of the data source like a headless CMS. An example `updatePost` mutation is outlined below. + +When you run a mutation on a post, a subscription event is published, which lets the plugin know it should respond and update nodes. The following mutation can be copied into the left side of the GraphQL playground so long as you replace "post-id" with a value returned for an ID from a query (like the one above). + +```graphql +mutation { + updatePost(id: "post-id", description: "Some data!") { + id + slug + description + } +} +``` + +The website's homepage will update with any changes while the source plugin is subscribed to changes, which is when the `preview: true` is provided in the example site's `gatsby-config`. + +You can also optionally listen for subscription events with this query in the playground which will display data when a mutation is run: + +```graphql +subscription { + posts { + id + description + } +} +``` + +A similar subscription is registered when the plugin is run, so you can also see subscription events logged when the plugin is running. diff --git a/examples/creating-source-plugins/api/README.md b/examples/creating-source-plugins/api/README.md new file mode 100644 index 0000000000000..c9fa5261a5770 --- /dev/null +++ b/examples/creating-source-plugins/api/README.md @@ -0,0 +1,3 @@ +# Example API + +A small GraphQL server with in-memory data, powered by [graphql-yoga](https://github.com/graphcool/graphql-yoga) 🧘. See the root of the monorepo for details about running this API alongisde the `example-site` and `source-plugin`. diff --git a/examples/creating-source-plugins/api/package.json b/examples/creating-source-plugins/api/package.json new file mode 100644 index 0000000000000..48e9f8626d1e9 --- /dev/null +++ b/examples/creating-source-plugins/api/package.json @@ -0,0 +1,43 @@ +{ + "name": "api", + "description": "A simple GraphQL server example with in-memory data", + "version": "1.0.0", + "license": "MIT", + "homepage": "https://general-repair.glitch.me", + "author": { + "name": "Risan Bagja Pradana", + "email": "risanbagja@gmail.com", + "url": "https://risan.io" + }, + "main": "src/index.js", + "repository": { + "type": "git", + "url": "git+https://github.com/risan/simple-graphql-server-example.git" + }, + "bugs": { + "url": "https://github.com/risan/simple-graphql-server-example/issues" + }, + "keywords": [ + "graphql", + "graphql-server", + "graphql-yoga" + ], + "scripts": { + "start": "node src/index.js", + "lint": "eslint *.js src", + "lint-fix": "eslint *.js src --fix" + }, + "dependencies": { + "dotenv": "^5.0.1", + "graphql-yoga": "^1.8.2", + "uniqid": "^4.1.1" + }, + "devDependencies": { + "eslint": "^4.19.1", + "eslint-config-airbnb-base": "^12.1.0", + "eslint-config-prettier": "^2.9.0", + "eslint-plugin-import": "^2.10.0", + "eslint-plugin-prettier": "^2.6.0", + "prettier": "^1.11.1" + } +} diff --git a/examples/creating-source-plugins/api/src/index.js b/examples/creating-source-plugins/api/src/index.js new file mode 100644 index 0000000000000..70b3e159cfa73 --- /dev/null +++ b/examples/creating-source-plugins/api/src/index.js @@ -0,0 +1,138 @@ +require("dotenv").config() +const { GraphQLServer, PubSub } = require("graphql-yoga") +const uniqid = require("uniqid") + +const CREATED = "created" +const UPDATED = "updated" +const DELETED = "deleted" + +const authors = [ + { + id: 1, + name: "Jay Gatsby", + }, + { + id: 2, + name: "Daisy Buchanan", + }, +] + +const posts = [ + { + id: uniqid(), + slug: "hello-world", + description: "Our first post on our site.", + imgUrl: "https://images.unsplash.com/photo-1534432586043-ead5b99229fb", + imgAlt: "Pug in a sweater", + authorId: 1, + }, + { + id: uniqid(), + slug: "company-vision", + description: "Our vision for a welcoming company.", + imgUrl: "https://images.unsplash.com/photo-1530041539828-114de669390e", + imgAlt: "Pug in a rainjacket", + authorId: 1, + }, + { + id: uniqid(), + slug: "redesigning-our-logo", + description: "What went into the new logo.", + imgUrl: "https://images.unsplash.com/photo-1541364983171-a8ba01e95cfc", + imgAlt: "Pug in glasses", + authorId: 2, + }, +] + +const resolvers = { + Query: { + info: () => "A simple GraphQL server example with in-memory data.", + posts: () => posts, + authors: () => authors, + }, + + Mutation: { + createPost: (root, { slug, description }) => { + const post = { + id: uniqid(), + slug, + description, + imgUrl: "https://images.unsplash.com/photo-1534432586043-ead5b99229fb", + imgAlt: "pug in a sweater", + authorId: 1, + } + + posts.push(post) + pubsub.publish(CREATED, { posts: [{ status: CREATED, ...post }] }) + + return post + }, + + updatePost: (root, { id, description }) => { + const postIdx = posts.findIndex(p => id === p.id) + + if (postIdx === null) { + return null + } + + posts[postIdx] = { ...posts[postIdx], description } + pubsub.publish(UPDATED, { + posts: [{ status: UPDATED, ...posts[postIdx] }], + }) + + return posts[postIdx] + }, + + deletePost: (root, { id }) => { + const postIdx = posts.findIndex(p => id === p.id) + + if (postIdx === null) { + return null + } + + const post = posts[postIdx] + pubsub.publish(DELETED, { + posts: [{ status: DELETED, ...posts[postIdx] }], + }) + + posts.splice(postIdx, 1) + + return post + }, + }, + + Post: { + id: root => root.id, + slug: root => root.slug, + description: root => root.description, + author: root => authors.find(author => author.id === root.authorId), + }, + + Author: { + id: root => root.id, + name: root => root.name, + }, + + Subscription: { + posts: { + subscribe: (parent, args, { pubsub }) => { + return pubsub.asyncIterator([CREATED, UPDATED, DELETED]) + }, + }, + }, +} + +const pubsub = new PubSub() +const server = new GraphQLServer({ + typeDefs: "./src/schema.graphql", + resolvers, + context: { pubsub }, +}) + +server.start( + { + port: + (process.env.PORT ? parseInt(process.env.PORT, 10) : undefined) || 4000, + }, + ({ port }) => console.log(`🏃🏻‍ Server is running on port ${port}.`) +) diff --git a/examples/creating-source-plugins/api/src/schema.graphql b/examples/creating-source-plugins/api/src/schema.graphql new file mode 100644 index 0000000000000..2dd1f9c5a67c6 --- /dev/null +++ b/examples/creating-source-plugins/api/src/schema.graphql @@ -0,0 +1,30 @@ +type Query { + info: String! + posts: [Post!]! + authors: [Author!]! +} + +type Mutation { + createPost(slug: String!, description: String!): Post! + updatePost(id: ID!, description: String!): Post + deletePost(id: ID!): Post +} + +type Post { + id: ID! + slug: String! + description: String! + imgUrl: String! + imgAlt: String! + author: Author! + status: String +} + +type Author { + id: ID! + name: String! +} + +type Subscription { + posts: [Post!]! +} diff --git a/examples/creating-source-plugins/example-site/README.md b/examples/creating-source-plugins/example-site/README.md new file mode 100644 index 0000000000000..5c03365ce2895 --- /dev/null +++ b/examples/creating-source-plugins/example-site/README.md @@ -0,0 +1,3 @@ +# Example Site + +See the root of the monorepo for details about running this site with the example `source-plugin` installed inside it. diff --git a/examples/creating-source-plugins/example-site/gatsby-config.js b/examples/creating-source-plugins/example-site/gatsby-config.js new file mode 100644 index 0000000000000..1d16c1582500d --- /dev/null +++ b/examples/creating-source-plugins/example-site/gatsby-config.js @@ -0,0 +1,22 @@ +/** + * Configure your Gatsby site with this file. + * + * See: https://www.gatsbyjs.org/docs/gatsby-config/ + */ + +module.exports = { + plugins: [ + // loads the source-plugin + { + resolve: `source-plugin`, + options: { + spaceId: "123", + preview: true, + cacheResponse: false, + }, + }, + // required to generate optimized images + `gatsby-plugin-sharp`, + `gatsby-transformer-sharp`, + ], +} diff --git a/examples/creating-source-plugins/example-site/package.json b/examples/creating-source-plugins/example-site/package.json new file mode 100644 index 0000000000000..51dc41c4900d2 --- /dev/null +++ b/examples/creating-source-plugins/example-site/package.json @@ -0,0 +1,34 @@ +{ + "name": "example-site", + "private": true, + "description": "A simplified bare-bones starter for Gatsby", + "version": "0.1.0", + "license": "MIT", + "scripts": { + "build": "gatsby build", + "develop": "gatsby develop", + "format": "prettier --write \"**/*.{js,jsx,json,md}\"", + "start": "npm run develop", + "serve": "gatsby serve", + "clean": "gatsby clean", + "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1" + }, + "dependencies": { + "gatsby": "^2.19.45", + "gatsby-image": "^2.3.1", + "gatsby-plugin-sharp": "^2.5.3", + "gatsby-transformer-sharp": "^2.4.3", + "react": "^16.12.0", + "react-dom": "^16.12.0" + }, + "devDependencies": { + "prettier": "^1.19.1" + }, + "repository": { + "type": "git", + "url": "https://github.com/gatsbyjs/gatsby-starter-hello-world" + }, + "bugs": { + "url": "https://github.com/gatsbyjs/gatsby/issues" + } +} diff --git a/examples/creating-source-plugins/example-site/src/pages/index.js b/examples/creating-source-plugins/example-site/src/pages/index.js new file mode 100644 index 0000000000000..e8491f64ec941 --- /dev/null +++ b/examples/creating-source-plugins/example-site/src/pages/index.js @@ -0,0 +1,72 @@ +import React from "react" +import { graphql } from "gatsby" +import Img from "gatsby-image" + +export default ({ data }) => ( +
+

Posts

+
+ {data.allPost.nodes.map(post => ( +
+

{post.slug}

+ By: {post.author.name} +

{post.description}

+ {post.imgAlt} +
+ ))} +
+
+) + +export const query = graphql` + { + allPost { + nodes { + id + slug + description + imgAlt + author { + id + name + } + slug + remoteImage { + id + childImageSharp { + id + fluid { + ...GatsbyImageSharpFluid + } + } + } + } + } + } +` diff --git a/examples/creating-source-plugins/example-site/static/favicon.ico b/examples/creating-source-plugins/example-site/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..1a466ba8852cf099ae40335e878897bde72e1735 GIT binary patch literal 2813 zcmV7wtO>0C#@O9*BXj0vnU3Txw<*2nLRVINq?LYz0THvB9>K0QA`=kA45 zH>rZGmlV#qnH$uCDPe+;0o=3Sp?kdq2WgKYI^SXKIZBYn-k+Edd<+6KvLh|TKS!{V zpmkAJ1tGJUk)?NIbT2}HbKK*+590b+#Ctc)-(n5gtDWwb65MI+Khl=Qpa}`SObc0D zcb3qzwnfCnoGI(U#i&M#k<6JI2B@4&y3QodXL{VnHQ%G>xde1yh{Cr~tOY{ox`SZ3 zFw9dcCna73B99M6W#~MX2swyNG~(c5oRRa8b|0Nk?#}ws8a4BarE2z<*Qz=9u2pl# zuVeW2SIYPvp5Yz5D+FN;y;dZLqypy9bp!1;&a#cc5n;b5oF_sa0%!7BRs4v9i?i=p zqX3V8cZ(`5*sFGw9ait`JF6NSE~)dUuQ7aZ^s+i!b5`wij;J+-`_z)}RWL!b=sS^| zO^BzKp+16XqQW+A10pY+rv>N1_&lQo@}62?LqfovcVD?$mA9J_zImft4fg10?@>o; zo7Cz>d(`}i5l|h2y!IA=7#7ARw4n>m6=zqgM&hh} z#G>&5kq^Fe?%k`ZWZ`a>$fLA2_RvJIZRJnhVu%oRO7cz?Wuo&BX|5aObF;!O>9V`< z{`Km>wvz^dXe6dw*G+Ku;~v#~qD3A4N0WN%_0#ID*G{p7nh4QbZGI`?TR9HxJfjvp z_#)d5_R>VKH`9!KL_3wy;w62^Lqz%k8LZQx2{6b*WtRSUtGd{D-EcA<^SIg5*{2Tf zcwdz+-m8kfvzgU`4OiO-u8F&<@LQYI3;BE0KdVlu&YQjbts8fZR4#wmu2%fKlE1Nt zh~;>08um9qVuc|h81)cb5(Z1iYkhzE7q1#U@9T{k&ed&wPd)j=m)W~$50h}6aH*bK z<$E9k>WBA!^Zg1{v-y~cX(q^;UAKDGA7{Mo5s^@*sFuPpg3(EKkTBZN677#8qVrIj zr9b+cx_P~eH;CwsuAFaIf0^@lHRJA5HIH0nu1405zNrB3;N5xmt>bT>nO5fMajV)(PZ_$2v<7h#4o znmJ7042Zk{znjUb$;OL4jve^WlSGfV$Mq{6>alNbV#7{Ea%NBu)ZAnU7WY@`AQBNj z4+QsDp74qIeL^13nd@2MAt9cM5Uc^UgGBzbzu9Nh3m@2^vM2HSI!=TlT0_FNEl6V4 zD%=-8zjFRg&tSElUcKo@{>d9AtKW=au9*NMa?WGi7G|C1kicPHsS+1 ze8-skgyi!%yL`TS|Enj}OftH3BOkO2R+&ueUedcxOh@PjyBO)#qG79to2%d5==o_1 zQoUUfwTxl_60@Ni?2w6=(jW3h3LpfwIv&P59X^)M9J-My7JYlOYQNTL461%tqx#jD z)hymm3;7{{Q6nNE4-zOY(B4Np&#K&AzP#lIxwL1dRdE0suIEV;#5|@u+d)9Bj&d;( z9XO3#hHaoPW^$ZAJiIO0i2pwl5O351U)rSeo;Y12rDc}}K+0lhAY!<&!y-gw zc-@1r6)e2qLQ$BB5J?y!hA9boA7^3IU;qiC^{oUpH^V~Ql}(U6B2Wnt>6V1}{l(wd zIA|#N+KV;=1lI%sILpGt3#vJoAVOxRTN1QB+%Zz~>z8k_w`*(ZRPDEc8&^A2C+&q4 z2DUkWQZfNuMyF_T6cnhms;Pbz*V&&Wp8n;~hya7;jw@5kfBFj70)ss+P(pb-2{ell zOB6mt#R-ne=l;df*CrxK5<^rcApq&}KnH@TxbpW7UEs{sO4*GO*NGcN0f8i{>V>z> zBHNy?=e#|gl91|b+p2o?8tqk8@;2`Yz;E2&vF@MX|sG!3A}Z@m1jshJ^(Wk1=DB2E!+sLOz&I+iC<5BC1M~iAaZwl>j

}uA3 zJBZ51Gh-{LePn)666(D9~}hFkGgt_C%~*Ee_( zDUfKEOKU|MTSzV78qq#Xmn8DYRpK2OEK&-wH9;QdKh4^XnBZF7Io5r;E)KR-OrsQI z9TdyjBSlSLls5hI?Cq+jD`JQU+C(ZBn@SR>?52OAgnOtv%a52wmZ%MoM>46aKhs*& zf$I!if5vXeCC$O;JPyMJJ&B3gM1X_W{(CdwUQ>9dfh5sJ*(HLM?dPTQfoI@+0Qq>r zW+F=s+jHQ+bf!=b7hHS50mFt}tS#Kf<0Jc!yruHFL!64|q$r9wvRi5NM`v%S z^5537F~eXjooD2=*I6MWfP}g`{f?NlIyr^NlXAQ?BdSMC>=htpc#pE88g_Tm@0SDh z#9ZBiG~TJklU#R4AE(F86SYhv{JfrQ8l8vAXa=qjo$q%}W*f7<7Mpm)8@z2w8uG!l zfsi>AFP`|pORDVg8ntJ`QFZjqW_AA5H69&td772zec~N7$#7pY!5h6M%G!ulUqH52 z@eub02b z+Kr!~dt$IYwGsRVm=!zr2hx5YP0yZ1f8NteQ7b2=%TZUS$Ka^R`2#7m8H8)Z^>jb1 z61kXDY`Mx&fw}eI|7zf=k&`s7$8Z7td9-7&sfe52je5pzBBsq<`(8}Ajc+x_dypPT zgtfiRfNC@N$$cpBw*zhpKSc;Ocu}%K|A`~a{hHJrwXjCILg(|&ab%VFCqSxyt=M)s zti%7`2_85H|86s$Hp-*(b~?BCC6FjW=(?58)u^QCdZLM@iD~@Ep>q9yx$uEL-Yj-E P00000NkvXXu0mjfF#vB* literal 0 HcmV?d00001 diff --git a/examples/creating-source-plugins/package.json b/examples/creating-source-plugins/package.json new file mode 100644 index 0000000000000..34071a17c59ee --- /dev/null +++ b/examples/creating-source-plugins/package.json @@ -0,0 +1,14 @@ +{ + "name": "creating-source-plugins", + "version": "1.0.0", + "description": "Monorepo for examples, api, and plugins for creating first class source plugins", + "main": "index.js", + "author": "@gillkyle", + "license": "MIT", + "workspaces": [ + "api", + "example-site", + "source-plugin" + ], + "private": true +} diff --git a/examples/creating-source-plugins/source-plugin/README.md b/examples/creating-source-plugins/source-plugin/README.md new file mode 100644 index 0000000000000..1d88f045e1984 --- /dev/null +++ b/examples/creating-source-plugins/source-plugin/README.md @@ -0,0 +1,3 @@ +# Example Source Plugin + +See the root of the monorepo for details about running this plugin inside of the `example-site` folder. It is installed in the example site and can be debugged and developed while running there. diff --git a/examples/creating-source-plugins/source-plugin/gatsby-node.js b/examples/creating-source-plugins/source-plugin/gatsby-node.js new file mode 100644 index 0000000000000..935297827e34f --- /dev/null +++ b/examples/creating-source-plugins/source-plugin/gatsby-node.js @@ -0,0 +1,274 @@ +const { createRemoteFileNode } = require(`gatsby-source-filesystem`) +const WebSocket = require("ws") +const { ApolloClient } = require("apollo-client") +const { InMemoryCache } = require("apollo-cache-inmemory") +const { split } = require("apollo-link") +const { HttpLink } = require("apollo-link-http") +const { WebSocketLink } = require("apollo-link-ws") +const { getMainDefinition } = require("apollo-utilities") +const fetch = require("node-fetch") +const gql = require("graphql-tag") + +/** + * ============================================================================ + * Create a GraphQL client to subscribe to live data changes + * ============================================================================ + */ + +// Create an http link: +const httpLink = new HttpLink({ + uri: "http://localhost:4000", + fetch, +}) + +// Create a WebSocket link: +const wsLink = new WebSocketLink({ + uri: `ws://localhost:4000`, + options: { + reconnect: true, + }, + webSocketImpl: WebSocket, +}) + +// using the ability to split links, you can send data to each link/url +// depending on what kind of operation is being sent +const link = split( + // split based on operation type + ({ query }) => { + const definition = getMainDefinition(query) + return ( + definition.kind === "OperationDefinition" && + definition.operation === "subscription" + ) + }, + wsLink, + httpLink +) + +const client = new ApolloClient({ + link, + cache: new InMemoryCache(), +}) + +/** + * ============================================================================ + * Helper functions and constants + * ============================================================================ + */ + +const POST_NODE_TYPE = `Post` +const AUTHOR_NODE_TYPE = `Author` + +// helper function for creating nodes +const createNodeFromData = (item, nodeType, helpers) => { + const nodeMetadata = { + id: helpers.createNodeId(`${nodeType}-${item.id}`), + parent: null, // this is used if nodes are derived from other nodes, a little different than a foreign key relationship, more fitting for a transformer plugin that is changing the node + children: [], + internal: { + type: nodeType, + content: JSON.stringify(item), + contentDigest: helpers.createContentDigest(item), + }, + } + + const node = Object.assign({}, item, nodeMetadata) + helpers.createNode(node) + return node +} + +/** + * ============================================================================ + * Verify plugin loads + * ============================================================================ + */ + +// should see message in console when running `gatsby develop` in example-site +exports.onPreInit = () => console.log("Loaded source-plugin") + +/** + * ============================================================================ + * Link nodes together with a customized GraphQL Schema + * ============================================================================ + */ + +exports.createSchemaCustomization = ({ actions }) => { + const { createTypes } = actions + createTypes(` + type Post implements Node { + id: ID! + slug: String! + description: String! + imgUrl: String! + imgAlt: String! + # create relationships between Post and File nodes for optimized images + remoteImage: File @link + # create relationships between Post and Author nodes + author: Author @link(from: "author.name" by: "name") + } + + type Author implements Node { + id: ID! + name: String! + }`) +} + +/** + * ============================================================================ + * Source and cache nodes from the API + * ============================================================================ + */ + +exports.sourceNodes = async function sourceNodes( + { + actions, + cache, + createContentDigest, + createNodeId, + getNodesByType, + getNode, + }, + pluginOptions +) { + const { createNode, touchNode, deleteNode } = actions + const helpers = Object.assign({}, actions, { + createContentDigest, + createNodeId, + }) + + // you can access plugin options here if need be + console.log(`Space ID: ${pluginOptions.spaceId}`) + + // simple caching example, you can find in .cache/caches/source-plugin/some-diskstore + await cache.set(`hello`, `world`) + console.log(await cache.get(`hello`)) + + // touch nodes to ensure they aren't garbage collected + getNodesByType(POST_NODE_TYPE).forEach(node => touchNode({ nodeId: node.id })) + getNodesByType(AUTHOR_NODE_TYPE).forEach(node => + touchNode({ nodeId: node.id }) + ) + + // listen for updates using subscriptions from the API + if (pluginOptions.preview) { + console.log( + "Subscribing to updates on ws://localhost:4000 (plugin is in Preview mode)" + ) + const subscription = await client.subscribe({ + query: gql` + subscription { + posts { + id + slug + description + imgUrl + imgAlt + author { + id + name + } + status + } + } + `, + }) + subscription.subscribe(({ data }) => { + console.log(`Subscription received:`) + console.log(data.posts) + data.posts.forEach(post => { + const nodeId = createNodeId(`${POST_NODE_TYPE}-${post.id}`) + switch (post.status) { + case "deleted": + deleteNode({ + node: getNode(nodeId), + }) + break + case "created": + case "updated": + default: + // created and updated can be handled by the same code path + // the post's id is presumed to stay constant (or can be inferred) + createNodeFromData(post, POST_NODE_TYPE, helpers) + break + } + }) + }) + } + + // store the response from the API in the cache + const cacheKey = "your-source-data-key" + let sourceData = await cache.get(cacheKey) + + // fetch fresh data if nothiing is found in the cache or a plugin option says not to cache data + if (!sourceData || !pluginOptions.cacheResponse) { + console.log("Not using cache for source data, fetching fresh content") + const { data } = await client.query({ + query: gql` + query { + posts { + id + slug + description + imgUrl + imgAlt + author { + id + name + } + } + authors { + id + name + } + } + `, + }) + await cache.set(cacheKey, data) + sourceData = data + } + + // loop through data returned from the api and create Gatsby nodes for them + sourceData.posts.forEach(post => + createNodeFromData(post, POST_NODE_TYPE, helpers) + ) + sourceData.authors.forEach(author => + createNodeFromData(author, AUTHOR_NODE_TYPE, helpers) + ) + + return +} + +/** + * ============================================================================ + * Transform remote file nodes + * ============================================================================ + */ + +exports.onCreateNode = async ({ + actions: { createNode }, + getCache, + createNodeId, + node, +}) => { + // transfrom remote file nodes using Gatsby sharp plugins + // because onCreateNode is called for all nodes, verify that you are only running this code on nodes created by your plugin + if (node.internal.type === POST_NODE_TYPE) { + // create a FileNode in Gatsby that gatsby-transformer-sharp will create optimized images for + const fileNode = await createRemoteFileNode({ + // the url of the remote image to generate a node for + url: node.imgUrl, + getCache, + createNode, + createNodeId, + parentNodeId: node.id, + }) + + if (fileNode) { + // used to add a field `remoteImage` to the Post node from the File node in the schemaCustomization API + node.remoteImage = fileNode.id + + // inference can link these without schemaCustomization like this, but creates a less sturdy schema + // node.remoteImage___NODE = fileNode.id + } + } +} diff --git a/examples/creating-source-plugins/source-plugin/index.js b/examples/creating-source-plugins/source-plugin/index.js new file mode 100644 index 0000000000000..172f1ae6a468c --- /dev/null +++ b/examples/creating-source-plugins/source-plugin/index.js @@ -0,0 +1 @@ +// noop diff --git a/examples/creating-source-plugins/source-plugin/package.json b/examples/creating-source-plugins/source-plugin/package.json new file mode 100644 index 0000000000000..53449361a87c5 --- /dev/null +++ b/examples/creating-source-plugins/source-plugin/package.json @@ -0,0 +1,28 @@ +{ + "name": "source-plugin", + "version": "1.0.0", + "description": "A minimal boilerplate for the essential files Gatsby looks for in a plugin", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [ + "gatsby", + "gatsby-plugin" + ], + "author": "Kyle Gill ", + "license": "MIT", + "dependencies": { + "apollo-cache-inmemory": "^1.6.5", + "apollo-client": "^2.6.8", + "apollo-link": "^1.2.13", + "apollo-link-http": "^1.5.16", + "apollo-link-ws": "^1.0.19", + "apollo-utilities": "^1.3.3", + "gatsby-source-filesystem": "^2.2.2", + "graphql": "^15.0.0", + "graphql-tag": "^2.10.3", + "node-fetch": "^2.6.0", + "ws": "^7.2.3" + } +}