feat: Live loading of external JS (converters/extensions)

[Nerivec]

Allows loading/unloading external converters and extensions in the same way, through MQTT and the file system (from data subdirs).

TODO

• feat: Support live loading of external converters zigbee-herdsman-converters#8332
• Tests
• Live loading of external JS zigbee2mqtt.io#3229
• Cleanup external_converters setting

[sjorge]

Seems after this PR the example the frontend generates no longer works as the settings and logger vars are no longer passed ? (based on the new exampels)

I also tried one of the example and it's not doing the publishes on mqtt either so I'm not sure if this is entirely broken now?

Old extension to work around fw bugs that used to work well:

/*
 * The newer Philips Hue with BT bulbs (by signify) support reporting, except for color_mode
 *
 * This extension will read the colorMode attribute if any of the known color attributes get reported.
 */
 class ColorModeReader {
    constructor(zigbee, mqtt, state, publishEntityState, eventBus, settings, logger) {
        this.zigbee = zigbee;
        this.mqtt = mqtt;
        this.state = state;
        this.publishEntityState = publishEntityState;
        this.eventBus = eventBus;
        this.settings = settings;
        this.logger = logger;
        this.colorAttributes = [
            'colorTemperature', 'currentX', 'currentY',
            'currentHue', 'enhancedCurrentHue',
            'currentSaturation',
        ];

    }

    async start() {
        this.logger.info('Starting color_mode helper');

        // attach to events
        this.eventBus.on('deviceMessage', this.onDeviceMessage.bind(this), this.constructor.name);
    }

    async stop() {
        this.eventBus.removeListeners(this.constructor.name);
    }

    onDeviceMessage(data) {
        // skip non interviewed devices
        if (!data.device.zh.interviewCompleted) return;

        // filter reporting of lightingColorCtrl
        if (
            (data.type == 'attributeReport') &&
            (data.cluster == 'lightingColorCtrl')
        ) {
            // filter on color related attributes
            for (const attrib of Object.keys(data.data)) {
                if (this.colorAttributes.includes(attrib)) {
                    // check if device has colorMode reporting
                    if (!this.hasColorModeReporting(data.device)) {
                        this.readColorMode(data.device, data.endpoint, Object.keys(data.data))
                    }
                    return;
                }
            }
        }
    }

    /**
     * Look for colorMode or enhancedColorMode reporting
     */
    hasColorModeReporting(device) {
        let ret = false;
        device.zh.endpoints.forEach(ep => {
            ep.configuredReportings.forEach(report => {
                // skip reports on non lightingColorCtrl cluster
                if (report.cluster.name != "lightingColorCtrl") return;

                // skip reports on non colorMode attributes
                if (!['colorMode', 'enhancedColorMode'].includes(report.attribute.name)) return;

                ret = true;
            });
        });
        return ret;
    }

    /**
     * Read colorMode and original attributes from report
     */
    readColorMode(device, endpoint, attributes=[]) {
        attributes.push('colorMode');
        this.logger.debug(`Reading '${attributes.join(', ')}' from '${device.name}' (${device.ieeeAddr})`);
        endpoint.read('lightingColorCtrl', attributes);
    }
}

module.exports = ColorModeReader;

After modifying based on the new examples in this PR:

/*
 * The newer Philips Hue with BT bulbs (by signify) support reporting, except for color_mode
 *
 * This extension will read the colorMode attribute if any of the known color attributes get reported.
 */
 class ColorModeReader {
    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
        this.zigbee = zigbee;
        this.mqtt = mqtt;
        this.state = state;
        this.publishEntityState = publishEntityState;
        this.eventBus = eventBus;
        //this.logger = logger;
        this.colorAttributes = [
            'colorTemperature', 'currentX', 'currentY',
            'currentHue', 'enhancedCurrentHue',
            'currentSaturation',
        ];
    }

    // no longer async ?
    start() {
        //this.logger.info('Starting color_mode helper');
        this.mqtt.publish('debug/log', 'Starting color_mode helper');

        // attach to events
        this.eventBus.on('deviceMessage', this.onDeviceMessage.bind(this), this.constructor.name);
    }

    stop() {
        this.mqtt.publish('debug/log', 'Stopping color_mode helper');
        //this.eventBus.removeListeners(this.constructor.name);
    }

    onDeviceMessage(data) {
        // skip non interviewed devices
        if (!data.device.zh.interviewCompleted) return;

        // filter reporting of lightingColorCtrl
        if (
            (data.type == 'attributeReport') &&
            (data.cluster == 'lightingColorCtrl')
        ) {
            // filter on color related attributes
            for (const attrib of Object.keys(data.data)) {
                if (this.colorAttributes.includes(attrib)) {
                    // check if device has colorMode reporting
                    if (!this.hasColorModeReporting(data.device)) {
                        this.readColorMode(data.device, data.endpoint, Object.keys(data.data))
                    }
                    return;
                }
            }
        }
    }

    /**
     * Look for colorMode or enhancedColorMode reporting
     */
    hasColorModeReporting(device) {
        let ret = false;
        device.zh.endpoints.forEach(ep => {
            ep.configuredReportings.forEach(report => {
                // skip reports on non lightingColorCtrl cluster
                if (report.cluster.name != "lightingColorCtrl") return;

                // skip reports on non colorMode attributes
                if (!['colorMode', 'enhancedColorMode'].includes(report.attribute.name)) return;

                ret = true;
            });
        });
        return ret;
    }

    /**
     * Read colorMode and original attributes from report
     */
    readColorMode(device, endpoint, attributes=[]) {
        attributes.push('colorMode');
        //this.logger.debug(`Reading '${attributes.join(', ')}' from '${device.name}' (${device.ieeeAddr})`);
        this.mqtt.publish('debug/log', `Reading '${attributes.join(', ')}' from '${device.name}' (${device.ieeeAddr})`);
        endpoint.read('lightingColorCtrl', attributes);
    }
}

module.exports = ColorModeReader;

This now 'loads' but doesn't seem to do anything, at all :|

[sjorge]

OK ignoring the example and checking the actual code change this works:

/*
 * The newer Philips Hue with BT bulbs (by signify) support reporting, except for color_mode
 *
 * This extension will read the colorMode attribute if any of the known color attributes get reported.
 */
 class ColorModeReader {

    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension, settings, logger) {
        this.zigbee = zigbee;
        this.mqtt = mqtt;
        this.state = state;
        this.publishEntityState = publishEntityState;
        this.eventBus = eventBus;
        this.logger = logger;
        this.colorAttributes = [
            'colorTemperature', 'currentX', 'currentY',
            'currentHue', 'enhancedCurrentHue',
            'currentSaturation',
        ];
    }

    async start() {
        this.logger.info('Starting color_mode helper');

        // attach to events
        this.eventBus.on('deviceMessage', this.onDeviceMessage.bind(this), this.constructor.name);
    }

    async stop() {
        this.logger.info('Stopping color_mode helper');
        this.eventBus.removeListeners(this.constructor.name);
    }

    onDeviceMessage(data) {
        // skip non interviewed devices
        if (!data.device.zh.interviewCompleted) return;

        // filter reporting of lightingColorCtrl
        if (
            (data.type == 'attributeReport') &&
            (data.cluster == 'lightingColorCtrl')
        ) {
            // filter on color related attributes
            for (const attrib of Object.keys(data.data)) {
                if (this.colorAttributes.includes(attrib)) {
                    // check if device has colorMode reporting
                    if (!this.hasColorModeReporting(data.device)) {
                        this.readColorMode(data.device, data.endpoint, Object.keys(data.data))
                    }
                    return;
                }
            }
        }
    }

    /**
     * Look for colorMode or enhancedColorMode reporting
     */
    hasColorModeReporting(device) {
        let ret = false;
        device.zh.endpoints.forEach(ep => {
            ep.configuredReportings.forEach(report => {
                // skip reports on non lightingColorCtrl cluster
                if (report.cluster.name != "lightingColorCtrl") return;

                // skip reports on non colorMode attributes
                if (!['colorMode', 'enhancedColorMode'].includes(report.attribute.name)) return;

                ret = true;
            });
        });
        return ret;
    }

    /**
     * Read colorMode and original attributes from report
     */
    readColorMode(device, endpoint, attributes=[]) {
        attributes.push('colorMode');
        this.logger.debug(`Reading '${attributes.join(', ')}' from '${device.name}' (${device.ieeeAddr})`);
        endpoint.read('lightingColorCtrl', attributes);
    }
}

module.exports = ColorModeReader;

So just a constructor change is needed:

constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension, settings, logger) {

Probably need to mention that in the release notes and fix the example that the frontend generates.

start/stop still seem to be async though, which is fine as that is already what the frontend generates.

[Nerivec]

It is mentioned in the External converters and extensions section and already is in the dev docs 😉
@Koenkk do you know where the frontend generation template is located?

[Koenkk]

@Nerivec it's here: https://github.com/nurikk/zigbee2mqtt-frontend/blob/dev/src/components/extensions-editor/example-extension.js