Merge pull request #1069 from NordicSemiconductor/feat/artifactory/ve…

…rsioned_sources.json

Add versioning to sources.json

package.json

@@ -96,7 +96,7 @@
     },
     "devDependencies": {
         "@electron/notarize": "^2.2.0",
-        "@nordicsemiconductor/pc-nrfconnect-shared": "^191.0.0",
+        "@nordicsemiconductor/pc-nrfconnect-shared": "^195.0.0",
         "@playwright/test": "^1.16.3",
         "@testing-library/user-event": "^14.4.3",
         "@types/chmodr": "1.0.0",

src/launcher/features/settings/Settings.tsx

@@ -143,12 +143,7 @@ const Sources = () => {
                                     variant="outline-secondary"
                                     size="sm"
                                     onClick={() =>
-                                        clipboard.writeText(
-                                            source.url.replace(
-                                                /source\.json$/,
-                                                'apps.json'
-                                            )
-                                        )
+                                        clipboard.writeText(source.url)
                                     }
                                     title="Copy URL to clipboard"
                                 >

src/main/apps/apps.ts

@@ -33,7 +33,7 @@ import {
     writeAppInfo,
 } from './app';
 import { downloadAppResources } from './appResource';
-import { maybeMigrateLegacyMetaFiles } from './legacyMetaFiles';
+import { maybeMigrateLegacyMetaFiles } from './dataMigration/legacyMetaFiles';
 import {
     downloadAllSources,
     getAllAppUrls,

src/main/apps/legacyMetaFiles.test.ts → src/main/apps/dataMigration/legacyMetaFiles.test.ts

@@ -12,7 +12,7 @@ import {
     createNewAppInfo,
 } from './legacyMetaFiles';
 
-jest.mock('../config', () => {
+jest.mock('../../config', () => {
     const { join } = jest.requireActual('path');
     const { homedir } = jest.requireActual('os');
     return {

src/main/apps/legacyMetaFiles.ts → src/main/apps/dataMigration/legacyMetaFiles.ts

@@ -11,15 +11,15 @@ import {
 import fs from 'fs-extra';
 import path from 'path';
 
-import { Source } from '../../ipc/sources';
-import { getAppsRootDir, getNodeModulesDir } from '../config';
-import { readFile, readJsonFile } from '../fileUtil';
-import { installedAppPath, writeAppInfo } from './app';
+import { Source } from '../../../ipc/sources';
+import { getAppsRootDir, getNodeModulesDir } from '../../config';
+import { readFile, readJsonFile } from '../../fileUtil';
+import { installedAppPath, writeAppInfo } from '../app';
 import {
     sourceJsonExistsLocally,
     writeSourceJson,
     writeWithdrawnJson,
-} from './sources';
+} from '../sources';
 
 type AppName = `pc-nrfconnect-${string}`;
 
@@ -96,7 +96,7 @@ export const createNewAppInfo = (
     };
 };
 
-export const createNewAppInfoForWithdrawnApp = (
+const createNewAppInfoForWithdrawnApp = (
     source: Source,
     appName: AppName,
     packageJson: PackageJsonLegacyApp,

src/main/apps/dataMigration/migrateSourcesJson.test.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2023 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-4-Clause
+ */
+
+import { existsSync } from 'fs';
+
+import { readSchemedJsonFile, writeSchemedJsonFile } from '../../fileUtil';
+import { migrateSourcesJson, oldSourcesJsonPath } from './migrateSourcesJson';
+
+jest.mock('fs');
+jest.mock('../../fileUtil');
+
+describe('migrating sources.json', () => {
+    const mockedExistsSync = jest.mocked(existsSync);
+    const mockedReadSchemedJsonFile = jest.mocked(readSchemedJsonFile);
+    const mockedWriteSchemedJsonFile = jest.mocked(writeSchemedJsonFile);
+
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+
+    it('should migrate sources.json to sources-versioned.json', () => {
+        const oldSourceJsonContent = {
+            source1: 'http://source1.com/apps.json',
+            source2: 'http://source2.com/source.json',
+        };
+        const newSourceVersionedJsonContent = {
+            v1: [
+                { name: 'source1', url: 'http://source1.com/source.json' },
+                { name: 'source2', url: 'http://source2.com/source.json' },
+            ],
+        };
+
+        // Arrange
+        mockedExistsSync.mockImplementation(
+            path => path === oldSourcesJsonPath()
+        );
+        mockedReadSchemedJsonFile.mockReturnValue(oldSourceJsonContent);
+
+        // Act
+        migrateSourcesJson();
+
+        // Assert
+        expect(mockedWriteSchemedJsonFile).toBeCalledWith(
+            expect.anything(),
+            expect.anything(),
+            newSourceVersionedJsonContent
+        );
+    });
+
+    it('should do nothing if sources.json does not exist', () => {
+        mockedExistsSync.mockReturnValue(false);
+
+        migrateSourcesJson();
+
+        expect(mockedReadSchemedJsonFile).not.toBeCalled();
+    });
+
+    it('should do nothing if sources-versioned.json already exists', () => {
+        mockedExistsSync.mockReturnValue(true);
+
+        migrateSourcesJson();
+
+        expect(mockedReadSchemedJsonFile).not.toBeCalled();
+    });
+});

src/main/apps/dataMigration/migrateSourcesJson.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2015 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-4-Clause
+ */
+
+import fs from 'fs-extra';
+import path from 'path';
+import { z } from 'zod';
+
+import { getAppsExternalDir } from '../../config';
+import { readSchemedJsonFile } from '../../fileUtil';
+import { sourcesVersionedJsonPath, writeSourcesFile } from '../sources';
+
+export const oldSourcesJsonPath = () =>
+    path.join(getAppsExternalDir(), 'sources.json');
+
+const oldSourcesJsonSchema = z.record(z.string(), z.string().url());
+
+export const migrateSourcesJson = () => {
+    if (
+        !fs.existsSync(oldSourcesJsonPath()) ||
+        fs.existsSync(sourcesVersionedJsonPath())
+    ) {
+        return;
+    }
+
+    const oldSourcesJson = readSchemedJsonFile(
+        oldSourcesJsonPath(),
+        oldSourcesJsonSchema
+    );
+    const migratedSources = Object.entries(oldSourcesJson).map(
+        ([name, url]) => ({
+            name,
+            url: url.replace(/apps\.json$/, 'source.json'),
+        })
+    );
+
+    writeSourcesFile(migratedSources);
+};

src/main/apps/sourceChanges.ts

@@ -34,11 +34,6 @@ const downloadSource = async (
     const sourceJson = await downloadSourceJson(url);
     const source: Source = { name: sourceJson.name, url };
 
-    const isLegacyUrl = url.endsWith('/apps.json') && sourceJson.apps == null;
-    if (isLegacyUrl) {
-        return downloadSource(url.replace(/apps\.json$/, 'source.json'));
-    }
-
     return { source, sourceJson };
 };
 

src/main/apps/sources.test.ts

@@ -6,6 +6,9 @@
 
 import { newWithdrawnJson } from './sources';
 
+jest.mock('fs');
+jest.mock('../fileUtil');
+
 test('computing the new list of withdrawn apps', () => {
     const oldWithdrawnJson = [
         'http://example.org/oldWithdrawnApp.json',

src/main/apps/sources.ts

@@ -6,11 +6,14 @@
 
 import {
     SourceJson,
+    sourceJsonSchema,
     WithdrawnJson,
+    withdrawnJsonSchema,
 } from '@nordicsemiconductor/pc-nrfconnect-shared/main';
 import { dialog } from 'electron';
 import fs from 'fs-extra';
 import path from 'path';
+import { z } from 'zod';
 
 import { SourceWithError } from '../../ipc/apps';
 import { OFFICIAL, Source, SourceName, SourceUrl } from '../../ipc/sources';
@@ -21,7 +24,7 @@ import {
     getNodeModulesDir,
 } from '../config';
 import describeError from '../describeError';
-import { readFile, readJsonFile, writeJsonFile } from '../fileUtil';
+import { readSchemedJsonFile, writeSchemedJsonFile } from '../fileUtil';
 import { ensureDirExists } from '../mkdir';
 import { downloadToJson } from '../net';
 
@@ -33,57 +36,48 @@ const officialSource = {
     url: 'https://developer.nordicsemi.com/.pc-tools/nrfconnect-apps/source.json',
 };
 
-const sourcesJsonPath = () => path.join(getAppsExternalDir(), 'sources.json');
+export const sourcesVersionedJsonPath = () =>
+    path.join(getAppsExternalDir(), 'sources-versioned.json');
 
-const convertToOldSourceJsonFormat = (allSources: Source[]) =>
-    Object.fromEntries(
-        allSources.map(source => [
-            source.name,
-            source.url.replace(/source\.json$/, 'apps.json'),
-        ])
-    );
-
-const convertFromOldSourceJsonFormat = (
-    sourceJsonParsed: Record<SourceName, SourceUrl>
-) =>
-    Object.entries(sourceJsonParsed).map(([name, url]) => ({
-        name,
-        url: url.replace(/apps\.json$/, 'source.json'),
-    }));
+const sourcesVersionedSchema = z.object({
+    v1: z.array(
+        z.object({
+            name: z.string(),
+            url: z.string().url(),
+        })
+    ),
+});
 
 const loadAllSources = () => {
-    if (!fs.existsSync(sourcesJsonPath())) {
+    if (!fs.existsSync(sourcesVersionedJsonPath())) {
         return [];
     }
-    let sourceJsonContent: string | undefined;
     try {
-        sourceJsonContent = readFile(sourcesJsonPath());
-        const sourceJsonParsed = JSON.parse(sourceJsonContent);
-
-        if (Array.isArray(sourceJsonParsed)) {
-            return sourceJsonParsed;
-        }
-        if (sourceJsonParsed != null && typeof sourceJsonParsed === 'object') {
-            return convertFromOldSourceJsonFormat(sourceJsonParsed);
-        }
-
-        throw new Error('Unable to parse `source.json`.');
+        return readSchemedJsonFile(
+            sourcesVersionedJsonPath(),
+            sourcesVersionedSchema
+        ).v1;
     } catch (err) {
         dialog.showErrorBox(
             'Could not load list of locally known sources',
             'No sources besides the official and the local one will be shown. ' +
                 'Also apps from other sources will be hidden.\n\nError: ' +
-                `${describeError(err)}\n\n` +
-                `Content of \`source.json\`+ \`${sourceJsonContent}\``
+                `${describeError(err)}`
         );
+        return [];
     }
-    return [];
+};
+
+export const writeSourcesFile = (allSources: Source[]) => {
+    writeSchemedJsonFile(sourcesVersionedJsonPath(), sourcesVersionedSchema, {
+        v1: allSources,
+    });
 };
 
 const saveAllSources = () => {
     ensureSourcesAreLoaded();
 
-    writeJsonFile(sourcesJsonPath(), convertToOldSourceJsonFormat(sources));
+    writeSourcesFile(sources);
 };
 
 export const removeFromSourceList = (
@@ -108,7 +102,7 @@ export const addToSourceList = (sourceToAdd: Source, doSave = true) => {
     }
 };
 
-export const ensureSourcesAreLoaded = () => {
+const ensureSourcesAreLoaded = () => {
     if (!sourcesAreLoaded) {
         sourcesAreLoaded = true;
 
@@ -140,19 +134,23 @@ export const downloadSourceJson = (sourceUrl: SourceUrl) =>
 
 const downloadSourceJsonToFile = async (source: Source) => {
     const sourceJson = await downloadSourceJson(source.url);
-    writeJsonFile(getSourceJsonPath(source), sourceJson);
+    writeSourceJson(source, sourceJson);
 
     return sourceJson;
 };
 
 const readSourceJson = (source: Source) =>
-    readJsonFile<SourceJson>(getSourceJsonPath(source), {
+    readSchemedJsonFile(getSourceJsonPath(source), sourceJsonSchema, {
         name: source.name,
         apps: [],
     });
 
 export const writeSourceJson = (source: Source, sourceJson: SourceJson) =>
-    writeJsonFile(getSourceJsonPath(source), sourceJson);
+    writeSchemedJsonFile(
+        getSourceJsonPath(source),
+        sourceJsonSchema,
+        sourceJson
+    );
 
 export const getAppUrls = (source: Source) => readSourceJson(source).apps;
 
@@ -168,12 +166,17 @@ const getWithdrawnJsonPath = (source: Source) =>
     path.join(getAppsRootDir(source.name), 'withdrawn.json');
 
 const readWithdrawnJson = (source: Source) =>
-    readJsonFile<WithdrawnJson>(getWithdrawnJsonPath(source), []);
+    readSchemedJsonFile(getWithdrawnJsonPath(source), withdrawnJsonSchema, []);
 
 export const writeWithdrawnJson = (
     source: Source,
     withdrawnJson: WithdrawnJson
-) => writeJsonFile(getWithdrawnJsonPath(source), withdrawnJson);
+) =>
+    writeSchemedJsonFile(
+        getWithdrawnJsonPath(source),
+        withdrawnJsonSchema,
+        withdrawnJson
+    );
 
 export const isInListOfWithdrawnApps = (
     source: SourceName,