New Action to transform entries to another content type

examples/22-transform-entries-to-type.js

@@ -0,0 +1,37 @@
+const MurmurHash3 = require('imurmurhash');
+
+// Basic example: create new content type.
+module.exports = function (migration) {
+  // please run 01-angry-dog first, add some entries
+  // create new content type
+  const copycat = migration.createContentType('copycat').name('copy of dog').description('super friendly copy dog');
+
+  // add field
+  copycat.createField('woofs', {
+    name: 'woof woof woof',
+    type: 'Symbol',
+    required: true
+  });
+
+  // add entry title
+  copycat.displayField('woofs');
+
+  migration.transformEntriesToType({
+    sourceContentType: 'dog',
+    targetContentType: 'copycat',
+    from: ['woofs'],
+    to: ['woofs'],
+    shouldPublish: false,
+    updateReferences: true,
+    removeOldEntries: true,
+    identityKey: function (fields) {
+      const value = fields.woofs['en-US'].toString();
+      return MurmurHash3(value).result().toString();
+    },
+    transformEntryForLocale: function (fromFields, currentLocale) {
+      return {
+        woofs: `copy - ${fromFields.woofs[currentLocale]}`
+      };
+    }
+  });
+};

index.d.ts

@@ -192,8 +192,12 @@ export interface ITransformEntriesConfig {
    * The return value must be an object with the same keys as specified in to. Their values will be written to the respective entry fields for the current locale (i.e. {nameField: 'myNewValue'}). If it returns undefined, this the values for this locale on the entry will be left untouched.
   */
   transformEntryForLocale: (fromFields: ContentFields, currentLocale: string) => any,
-  /** (optional) – If true, the transformed entries will be published. If false, both will remain in draft state (default true) */
-  shouldPublish?: boolean
+  /** (optional) – If true, the transformed entries will be published. If false, they will remain in draft state. When the value is set to "preserve" items will be published only if the original entry was published as well (default true) */
+  shouldPublish?: boolean|"preserve",
+  /** (optional) – If true, references to the old entry are replaced with references to the new entry (default true) */
+  updateReferences?: boolean,
+  /** (optional) – If true, the original entry is removed after the new entry was created (default true) */
+  removeOldEntries?: boolean
 }
 
 export interface IDeriveLinkedEntriesConfig {
@@ -281,6 +285,12 @@ export default interface Migration {
    */
   transformEntries (transformation: ITransformEntriesConfig): void
 
+  /**
+   * For the given content type, transforms all its entries according to the user-provided transformEntryForLocale function into a new content type. For each entry, the CLI will call this function once per locale in the space, passing in the from fields and the locale as arguments. The transform function is expected to return an object with the desired target fields. If it returns undefined, this entry locale will be left untouched
+   * @param transformation 
+   */
+  transformEntriesToType (transformation: ITransformEntriesConfig): void
+
   /**
    * For each entry of the given content type (source entry), derives a new entry and sets up a reference to it on the source entry. The content of the new entry is generated by the user-provided deriveEntryForLocale function. For each source entry, this function will be called as many times as there are locales in the space. Each time, it will be called with the from fields and one of the locales as arguments. The derive function is expected to return an object with the desired target fields. If it returns undefined, the new entry will have no values for the current locale.
    * @param transformation 

src/lib/action/entry-transform-to-type.ts

@@ -0,0 +1,123 @@
+import EntryTransformToType from '../interfaces/entry-transform-to-type'
+import { APIAction } from './action'
+import { OfflineAPI } from '../offline-api'
+import Entry from '../entities/entry'
+import * as _ from 'lodash'
+
+class EntryTransformToTypeAction extends APIAction {
+  private fromFields?: string[]
+  private sourceContentTypeId: string
+  private targetContentTypeId: string
+  private transformEntryForLocale: (inputFields: any, locale: string) => Promise<any>
+  private identityKey: (fromFields: any) => Promise<string>
+  private shouldPublish: boolean|'preserve'
+  private removeOldEntries: boolean
+  private updateReferences: boolean
+
+  constructor (entryTransformation: EntryTransformToType) {
+    super()
+    this.fromFields = entryTransformation.from
+    this.sourceContentTypeId = entryTransformation.sourceContentType
+    this.targetContentTypeId = entryTransformation.targetContentType
+    this.identityKey = entryTransformation.identityKey
+    this.shouldPublish = entryTransformation.shouldPublish || false
+    this.removeOldEntries = entryTransformation.removeOldEntries || false
+    this.updateReferences = entryTransformation.updateReferences || false
+    this.transformEntryForLocale = entryTransformation.transformEntryForLocale
+  }
+
+  async applyTo (api: OfflineAPI) {
+    const entries: Entry[] = await api.getEntriesForContentType(this.sourceContentTypeId)
+    const locales: string[] = await api.getLocalesForSpace()
+
+    for (const entry of entries) {
+      const inputs = this.fromFields ? _.pick(entry.fields, this.fromFields) : entry.fields
+      const newEntryId = await this.identityKey(inputs)
+      const hasEntry = await api.hasEntry(newEntryId)
+
+      let skipEntry = true
+      let fieldsForTargetEntry = {}
+
+      for (const locale of locales) {
+        let outputsForCurrentLocale
+        try {
+          outputsForCurrentLocale = await this.transformEntryForLocale(inputs, locale)
+        } catch (err) {
+          await api.recordRuntimeError(err)
+          continue
+        }
+
+        if (outputsForCurrentLocale === undefined) {
+          continue
+        }
+
+        skipEntry = false
+
+        // we collect all the values for the target entry before writing it to the
+        // offline API because we don't yet know if the entry might be skipped
+        // TODO: verify that the derivedFields actually get written to
+        // and to no other field
+        for (const [fieldId, localizedValue] of _.entries(outputsForCurrentLocale)) {
+          if (!fieldsForTargetEntry[fieldId]) {
+            fieldsForTargetEntry[fieldId] = {}
+          }
+          fieldsForTargetEntry[fieldId][locale] = localizedValue
+        }
+      }
+
+      // if derive returned undefined for all locales of this entry, there are no changes
+      // to be made, neither on the source nor the target entry, so we move on to the next
+      if (skipEntry) {
+        continue
+      }
+
+      if (!hasEntry) {
+        const targetEntry = await api.createEntry(this.targetContentTypeId, newEntryId)
+
+        // we are not skipping this source entry and the target entry does not yet exist,
+        // so now is the time to write the collected target entry values to the offline API
+        for (const [fieldId, localizedField] of _.entries(fieldsForTargetEntry)) {
+          if (!targetEntry.fields[fieldId]) {
+            targetEntry.setField(fieldId, {})
+          }
+
+          for (const [locale, localizedValue] of _.entries(localizedField)) {
+            targetEntry.setFieldForLocale(fieldId, locale, localizedValue)
+          }
+
+        }
+        await api.saveEntry(targetEntry.id)
+        if (this.shouldPublish === true || (this.shouldPublish === 'preserve' && entry.isPublished) ) {
+          await api.publishEntry(targetEntry.id)
+        }
+      }
+
+      // look for entries linking to the old entry and replace them with references to the new entry
+      if (this.updateReferences) {
+        const links = await api.getLinks(entry.id, locales)
+        for (const link of links) {
+          if (!link.isInArray()) {
+            link.element.setFieldForLocale(link.field, link.locale,{ sys: { id: newEntryId, type: 'Link', linkType: 'Entry'}})
+          } else {
+            link.element.replaceArrayLinkForLocale(link.field, link.locale, link.index, newEntryId)
+          }
+
+          await api.saveEntry(link.element.id)
+          if (this.shouldPublish === true || (this.shouldPublish === 'preserve' && link.element.isPublished) ) {
+            await api.publishEntry(link.element.id)
+          }
+        }
+      }
+
+      // remove the original item
+      if (this.removeOldEntries) {
+        if (entry.isPublished) {
+          await api.unpublishEntry(entry.id)
+        }
+        await api.deleteEntry(entry.id)
+      }
+    }
+  }
+}
+
+export { EntryTransformToTypeAction }

src/lib/entities/entry.ts

@@ -1,17 +1,20 @@
 import { cloneDeep } from 'lodash'
 import APIEntry from '../interfaces/api-entry'
+import { isNullOrUndefined } from 'util'
 
 class Entry {
   private _id: string
   private _contentTypeId: string
   private _version: number
   private _fields: object
+  private _isPublished: boolean
 
   constructor (entry: APIEntry) {
     this._id = entry.sys.id
     this._fields = entry.fields
     this._version = entry.sys.version
     this._contentTypeId = entry.sys.contentType.sys.id
+    this._isPublished = !isNullOrUndefined(entry.sys.publishedVersion)
   }
 
   get id () {
@@ -40,6 +43,18 @@ class Entry {
     this._fields[id] = field
   }
 
+  replaceArrayLinkForLocale (id: string, locale: string, index: number, linkId: string) {
+    const link = { sys: { id: linkId, type: 'Link', linkType: 'Entry'}}
+    const field = this._fields[id] || {}
+    const fieldArray = field[locale] as Array<any> || new Array<any>()
+
+    if (fieldArray.length < index + 1) {
+      fieldArray.push(link)
+    } else {
+      fieldArray.splice(index, 1, link)
+    }
+  }
+
   get version () {
     return this._version
   }
@@ -48,6 +63,14 @@ class Entry {
     this._version = version
   }
 
+  get isPublished () {
+    return this._isPublished
+  }
+
+  set isPublished (published: boolean) {
+    this._isPublished = published
+  }
+
   toApiEntry (): APIEntry {
     const sys = {
       id: this.id,

src/lib/entities/link.ts

@@ -0,0 +1,44 @@
+import Entry from './entry'
+
+class Link {
+  private _field: string
+  private _locale: string
+  private _index: number
+  private _element: Entry
+
+  constructor (element: Entry, field: string, locale: string, index: number = -1) {
+    this._field = field
+    this._locale = locale
+    this._index = index
+    this._element = element
+  }
+
+  get field (): string {
+    return this._field
+  }
+
+  get locale (): string {
+    return this._locale
+  }
+
+  get index (): number {
+    return this._index
+  }
+
+  get element (): Entry {
+    return this._element
+  }
+
+  isValid (): boolean {
+    return this._field.length > 0
+  }
+
+  isInArray (): boolean {
+    return this._index !== -1
+  }
+}
+
+export {
+  Link as default,
+  Link
+}

src/lib/fetcher.ts

@@ -14,23 +14,27 @@ export default class Fetcher implements APIFetcher {
   }
 
   async getEntriesInIntents (intentList: IntentList): Promise<APIEntry[]> {
+    const loadAllEntries = intentList.getIntents().some((intent) => intent.requiresAllEntries())
+
     const ids: string[] = _.uniq(
       intentList.getIntents()
-      .filter((intent) => intent.isContentTransform() || intent.isEntryDerive())
+      .filter((intent) => intent.isContentTransform() || intent.isEntryDerive() || intent.isEntryTransformToType())
       .map((intent) => intent.getContentTypeId())
     )
 
-    if (ids.length === 0) {
+    if (!loadAllEntries && ids.length === 0) {
       return []
     }
 
     let entries: APIEntry[] = []
     let skip: number = 0
 
+    const filterSpecification = loadAllEntries ? '' : `sys.contentType.sys.id[in]=${ids.join(',')}&`
+
     while (true) {
       const response = await this.makeRequest({
         method: 'GET',
-        url: `/entries?sys.contentType.sys.id[in]=${ids.join(',')}&skip=${skip}`
+        url: `/entries?${filterSpecification}skip=${skip}`
       })
 
       entries = entries.concat(response.items)

src/lib/intent-validator/entry-transform-to-type.ts

@@ -0,0 +1,33 @@
+import Intent from '../intent/base-intent'
+import SchemaValidator from './schema-validator'
+import * as Joi from 'joi'
+
+class EntryTransformToTypeIntentValidator extends SchemaValidator {
+  protected article = 'an'
+  protected displayName = 'entry derivation'
+
+  appliesTo (step: Intent) {
+    return step.isEntryDerive()
+  }
+
+  // NOTE: this could be change to return the object to
+  // validate against the schema instead of returning
+  // just the prop name
+  get propertyNameToValidate () {
+    return 'derivation'
+  }
+
+  get schema () {
+    return {
+      sourceContentType: Joi.string().required(),
+      targetContentType: Joi.string().required(),
+      from: Joi.array().items(Joi.string()).required(),
+      to: Joi.array().items(Joi.string()).required(),
+      identityKey: Joi.func().required(),
+      shouldPublish: Joi.boolean(),
+      deriveEntryForLocale: Joi.func().required()
+    }
+  }
+}
+
+export default EntryTransformToTypeIntentValidator

src/lib/intent/base-intent.ts

@@ -26,6 +26,10 @@ export default abstract class Intent implements IntentInterface {
     return this.payload.fieldId
   }
 
+  requiresAllEntries () {
+    return false
+  }
+
   isContentTypeUpdate () {
     return false
   }
@@ -66,6 +70,10 @@ export default abstract class Intent implements IntentInterface {
     return false
   }
 
+  isEntryTransformToType () {
+    return false
+  }
+
   isEditorInterfaceUpdate () {
     return false
   }

src/lib/intent/composed-intent.ts

@@ -89,6 +89,10 @@ export default class ComposedIntent implements Intent {
     return false
   }
 
+  isEntryTransformToType (): boolean {
+    return false
+  }
+
   getContentTypeId (): string {
     return this.contentTypeId
   }
@@ -97,6 +101,10 @@ export default class ComposedIntent implements Intent {
     return [this.getContentTypeId()]
   }
 
+  requiresAllEntries (): boolean {
+    return false
+  }
+
   groupsWith (): boolean {
     return false
   }