From cb0dd8908fb641be7cae01882ceb3fb8fa3b0a0a Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 27 Nov 2024 03:34:19 +0530 Subject: [PATCH 01/37] implement jsonata manager --- lib/config/options/index.ts | 21 ++ lib/modules/manager/custom/api.ts | 2 + lib/modules/manager/custom/index.spec.ts | 2 + .../manager/custom/jsonata/index.spec.ts | 256 ++++++++++++++++++ lib/modules/manager/custom/jsonata/index.ts | 89 ++++++ lib/modules/manager/custom/jsonata/readme.md | 161 +++++++++++ lib/modules/manager/custom/jsonata/types.ts | 26 ++ lib/modules/manager/custom/jsonata/utils.ts | 63 +++++ lib/modules/manager/custom/types.ts | 7 +- pnpm-lock.yaml | 1 - 10 files changed, 625 insertions(+), 3 deletions(-) create mode 100644 lib/modules/manager/custom/jsonata/index.spec.ts create mode 100644 lib/modules/manager/custom/jsonata/index.ts create mode 100644 lib/modules/manager/custom/jsonata/readme.md create mode 100644 lib/modules/manager/custom/jsonata/types.ts create mode 100644 lib/modules/manager/custom/jsonata/utils.ts diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 0575942785574a..92b9580dd6683d 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2749,6 +2749,27 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'matchQueries', + description: + 'JSON query to use. Valid only within a `customManagers` object of type `jsonata`.', + type: 'array', + subType: 'string', + parents: ['customManagers'], + cli: false, + env: false, + }, + { + name: 'matchStrings', + description: + 'Regex capture rule to use. Valid only within a `customManagers` object.', + type: 'array', + subType: 'string', + format: 'regex', + parents: ['customManagers'], + cli: false, + env: false, + }, { name: 'matchStringsStrategy', description: 'Strategy how to interpret matchStrings.', diff --git a/lib/modules/manager/custom/api.ts b/lib/modules/manager/custom/api.ts index de5e051ca72ca1..f7dc64aaef3cc6 100644 --- a/lib/modules/manager/custom/api.ts +++ b/lib/modules/manager/custom/api.ts @@ -1,7 +1,9 @@ import type { ManagerApi } from '../types'; +import * as jsonata from './jsonata'; import * as regex from './regex'; const api = new Map(); export default api; api.set('regex', regex); +api.set('jsonata', jsonata); diff --git a/lib/modules/manager/custom/index.spec.ts b/lib/modules/manager/custom/index.spec.ts index 54e922bd1ec517..61a6466dbcd447 100644 --- a/lib/modules/manager/custom/index.spec.ts +++ b/lib/modules/manager/custom/index.spec.ts @@ -10,6 +10,8 @@ describe('modules/manager/custom/index', () => { expect(customManager.isCustomManager('npm')).toBe(false); expect(customManager.isCustomManager('regex')).toBe(true); expect(customManager.isCustomManager('custom.regex')).toBe(false); + expect(customManager.isCustomManager('jsonata')).toBe(true); + expect(customManager.isCustomManager('custom.jsonata')).toBe(false); }); }); }); diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts new file mode 100644 index 00000000000000..8da9bc930bc6c7 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -0,0 +1,256 @@ +import { logger } from '../../../../logger'; +import type { JsonataExtractConfig } from './types'; +import { defaultConfig, extractPackageFile } from '.'; + +describe('modules/manager/custom/jsonata/index', () => { + it('has default config', () => { + expect(defaultConfig).toEqual({ + fileMatch: [], + }); + }); + + it('extracts data when no templates are used', async () => { + const json = ` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "http://brr.brr", + "dep_type": "dev" + } + ] + }`; + const config = { + matchQueries: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res?.deps).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( + 1, + ); + expect( + res?.deps.filter((dep) => dep.currentValue === '1.2.3'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === '1234'), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( + 1, + ); + expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( + 1, + ); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'custom-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.registryUrls?.includes('http://brr.brr/')), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + }); + + it('applies templates', async () => { + const json = ` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": "1.2.3", + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "http://brr.brr", + "dep_type": "dev" + }, + { + }] + }`; + const config = { + matchQueries: [ + `packages.{ + "depName": dep_name, + "packageName": package_name, + "currentValue": current_value, + "currentDigest": current_digest, + "datasource": data_source, + "versioning": versioning, + "extractVersion": extract_version, + "registryUrl": registry_url, + "depType": dep_type + }`, + ], + depNameTemplate: + '{{#if depName}}{{depName}}{{else}}default-dep-name{{/if}}', + packageNameTemplate: + '{{#if packageName}}{{packageName}}{{else}}default-package-name{{/if}}', + currentValueTemplate: + '{{#if currentValue}}{{currentValue}}{{else}}default-current-value{{/if}}', + currentDigestTemplate: + '{{#if currentDigest}}{{currentDigest}}{{else}}default-current-digest{{/if}}', + datasourceTemplate: + '{{#if datasource}}{{datasource}}{{else}}default-datasource{{/if}}', + versioningTemplate: + '{{#if versioning}}{{versioning}}{{else}}default-versioning{{/if}}', + extractVersionTemplate: + '{{#if extractVersion}}{{extractVersion}}{{else}}default-extract-version{{/if}}', + registryUrlTemplate: + '{{#if registryUrl}}{{registryUrl}}{{else}}http://default.registry.url{{/if}}', + depTypeTemplate: + '{{#if depType}}{{depType}}{{else}}default-dep-type{{/if}}', + }; + const res = await extractPackageFile(json, 'unused', config); + + expect(res?.deps).toHaveLength(2); + + expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( + 1, + ); + expect( + res?.deps.filter((dep) => dep.currentValue === '1.2.3'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === '1234'), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( + 1, + ); + expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( + 1, + ); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'custom-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.registryUrls?.includes('http://brr.brr/')), + ).toHaveLength(1); + expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + + expect( + res?.deps.filter((dep) => dep.depName === 'default-dep-name'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.packageName === 'default-package-name'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentValue === 'default-current-value'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.currentDigest === 'default-current-digest'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.datasource === 'default-datasource'), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.versioning === 'default-versioning'), + ).toHaveLength(1); + expect( + res?.deps.filter( + (dep) => dep.extractVersion === 'default-extract-version', + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => + dep.registryUrls?.includes('http://default.registry.url/'), + ), + ).toHaveLength(1); + expect( + res?.deps.filter((dep) => dep.depType === 'default-dep-type'), + ).toHaveLength(1); + }); + + it('returns null when content is not json', async () => { + jest.mock('renovate/lib/logger'); + const res = await extractPackageFile( + 'not-json', + 'foo-file', + {} as JsonataExtractConfig, + ); + expect(res).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + expect.anything(), + `error parsing 'foo-file'`, + ); + }); + + it('returns null if no dependencies found', async () => { + const config = { + matchQueries: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toBeNull(); + }); + + it('returns null if invalid template', async () => { + jest.mock('renovate/lib/logger'); + const config = { + matchQueries: [`{"depName": "foo"}`], + versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'Error compiling template for JSONata manager', + ); + }); + + it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { + jest.mock('renovate/lib/logger'); + const config = { + matchQueries: [`{"depName": "foo"}`], + registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res).not.toBeNull(); + expect(logger.warn).toHaveBeenCalledWith( + { value: 'this-is-not-a-valid-url-foo' }, + 'Invalid json manager registryUrl', + ); + }); + + it('extracts multiple dependencies with multiple matchQueries', async () => { + const config = { + matchQueries: [`{"depName": "foo"}`, `{"depName": "bar"}`], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res?.deps).toHaveLength(2); + }); + + it('extracts dependency with autoReplaceStringTemplate', async () => { + const config = { + matchQueries: [`{"depName": "foo"}`], + autoReplaceStringTemplate: 'auto-replace-string-template', + }; + const res = await extractPackageFile('{}', 'values.yaml', config); + expect(res?.autoReplaceStringTemplate).toBe('auto-replace-string-template'); + }); +}); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts new file mode 100644 index 00000000000000..a627efb00e8588 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -0,0 +1,89 @@ +import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; +import { logger } from '../../../../logger'; +import type { PackageDependency, PackageFile, Result } from '../../types'; +import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; +import { createDependency } from './utils'; + +export const supportedDatasources: string[] = ['*']; + +export const defaultConfig = { + fileMatch: [], +}; + +export interface CustomPackageFile extends PackageFile { + matchQueries: string[]; +} + +export function testJsonQuery(query: string): void { + jsonata(query); +} + +const validMatchFields = [ + 'depName', + 'packageName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', + 'depType', +]; + +function handleMatching( + json: unknown, + packageFile: string, + config: JsonataExtractConfig, +): PackageDependency[] { + return config.matchQueries + .flatMap((matchQuery) => jsonata(matchQuery).evaluate(json) ?? []) + .map((queryResult) => + createDependency(queryResult as Record, config), + ) + .filter(is.truthy); +} + +export function extractPackageFile( + content: string, + packageFile: string, + config: JsonataExtractConfig, +): Result { + let deps: PackageDependency[]; + + let json; + try { + json = JSON.parse(content); + } catch (err) { + logger.warn( + { err, content, fileName: packageFile }, + `error parsing '${packageFile}'`, + ); + return null; + } + + deps = handleMatching(json, packageFile, config); + + // filter all null values + deps = deps.filter(is.truthy); + if (deps.length) { + const res: CustomPackageFile & JSONataManagerTemplates = { + deps, + matchQueries: config.matchQueries, + }; + // copy over templates for autoreplace + for (const field of validMatchFields.map( + (f) => `${f}Template` as keyof JSONataManagerTemplates, + )) { + if (config[field]) { + res[field] = config[field]; + } + } + if (config.autoReplaceStringTemplate) { + res.autoReplaceStringTemplate = config.autoReplaceStringTemplate; + } + return res; + } + + return null; +} diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md new file mode 100644 index 00000000000000..e05956222ecb95 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -0,0 +1,161 @@ +The `json-jsonata` manager is designed to allow users to manually configure Renovate for how to find dependencies in JSON files that aren't detected by the built-in package managers. + +This manager is unique in Renovate in that: + +- It is configurable via [JSONata](https://jsonata.org/) queries. +- Through the use of the `jsonataManagers` config, multiple "JSONata managers" can be created for the same repository. +- It can extract any `datasource`. + +To configure it, use the following syntax: + +``` +{ + "jsonataManagers": [ + { + "fileMatch": [""], + "matchQueries": [''], + ... + } + ] +} +``` + +Where `` is a [JSONata](https://docs.jsonata.org/overview.html) query that transform the contents into a JSON object with the following schema: + +```json +{ + "depName": "", + "packageName": "", + "currentValue": "", + "currentDigest": "", + "datasource": "", + "versioning": "", + "extractVersion": "", + "registryUrl": "", + "depType": "" +} +``` + +The meaning of each field is the same as the meaning of the capturing groups for regex managers. + +The following configuration is also available for each `jsonManager` element, again with the same meaning as for the regex manager: + +- `depNameTemplate`. +- `packageNameTemplate`. +- `currentValueTemplate`. +- `currentDigestTemplate`. +- `datasourceTemplate`. +- `versioningTemplate`. +- `extractVersionTemplate`. +- `registryUrlTemplate`. +- `depTypeTemplate`. + +### Example queries + +Below are some example queries for the generic JSON manager. You can also use the [JSONata test website](https://try.jsonata.org) to experiment with queries. + +_Dependencies spread in different nodes, and we want to limit the extraction to a particular node:_ + +```json +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +production.{ "depName": package, "currentValue": version } +``` + +_Dependencies spread in different nodes, and we want to extract all of them as if they were in the same node:_ + +```json +{ + "production": [ + { + "version": "1.2.3", + "package": "foo" + } + ], + "development": [ + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +*.{ "depName": package, "currentValue": version } +``` + +_The dependency name is in a JSON node name and the version is in a child leaf to that node_: + +```json +{ + "foo": { + "version": "1.2.3" + }, + "bar": { + "version": "4.5.6" + } +} +``` + +Query: + +``` +$each(function($v, $n) { { "depName": $n, "currentValue": $v.version } }) +``` + +_The name of the dependency and the version are both value nodes of the same parent node:_ + +```json +{ + "packages": [ + { + "version": "1.2.3", + "package": "foo" + }, + { + "version": "4.5.6", + "package": "bar" + } + ] +} +``` + +Query: + +``` +packages.{ "depName": package, "currentValue": version } +``` + +_The name of the dependency and the version are in the same string:_ + +```json +{ + "packages": ["foo@1.2.3", "bar@4.5.6"] +} +``` + +Query: + +``` +$map($map(packages, function ($v) { $split($v, "@") }), function ($v) { { "depName": $v[0], "currentVersion": $v[1] } }) +``` diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts new file mode 100644 index 00000000000000..04832a4e6433b9 --- /dev/null +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -0,0 +1,26 @@ +import type { ExtractConfig } from '../../types'; + +export interface JSONataManagerTemplates { + depNameTemplate?: string; + packageNameTemplate?: string; + datasourceTemplate?: string; + versioningTemplate?: string; + depTypeTemplate?: string; + currentValueTemplate?: string; + currentDigestTemplate?: string; + extractVersionTemplate?: string; + registryUrlTemplate?: string; +} + +export interface JSONataManagerConfig extends JSONataManagerTemplates { + // fileMatch: string[]; + matchQueries: string[]; + autoReplaceStringTemplate?: string; +} + +export interface JsonataExtractConfig + extends ExtractConfig, + JSONataManagerTemplates { + autoReplaceStringTemplate?: string; + matchQueries: string[]; +} diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts new file mode 100644 index 00000000000000..ff0d93c658e01d --- /dev/null +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -0,0 +1,63 @@ +import { URL } from 'url'; +import { logger } from '../../../../logger'; +import * as template from '../../../../util/template'; +import type { PackageDependency } from '../../types'; +import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; + +export const validMatchFields = [ + 'depName', + 'packageName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', + 'depType', +] as const; + +type ValidMatchFields = (typeof validMatchFields)[number]; + +export function createDependency( + queryResult: Record, + config: JsonataExtractConfig, +): PackageDependency | null { + const dependency: PackageDependency = {}; + + function updateDependency(field: ValidMatchFields, value: string): void { + switch (field) { + case 'registryUrl': + // check if URL is valid and pack inside an array + try { + const url = new URL(value).toString(); + dependency.registryUrls = [url]; + } catch { + logger.warn({ value }, 'Invalid json manager registryUrl'); + } + break; + default: + dependency[field] = value; + break; + } + } + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template` as keyof JSONataManagerTemplates; + const tmpl = config[fieldTemplate]; + if (tmpl) { + try { + const compiled = template.compile(tmpl, queryResult, false); + updateDependency(field, compiled); + } catch { + logger.warn( + { template: tmpl }, + 'Error compiling template for JSONata manager', + ); + return null; + } + } else if (queryResult[field]) { + updateDependency(field, queryResult[field]); + } + } + return dependency; +} diff --git a/lib/modules/manager/custom/types.ts b/lib/modules/manager/custom/types.ts index de387685f80717..6e7b2595cc66cb 100644 --- a/lib/modules/manager/custom/types.ts +++ b/lib/modules/manager/custom/types.ts @@ -1,8 +1,11 @@ +import type { JSONataManagerConfig } from './jsonata/types'; import type { RegexManagerConfig } from './regex/types'; -export interface CustomExtractConfig extends Partial {} +export interface CustomExtractConfig + extends Partial, + Partial {} -export type CustomManagerName = 'regex'; +export type CustomManagerName = 'regex' | 'jsonata'; export interface CustomManager extends Partial { customType: CustomManagerName; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 72b31b02076765..fa286c47bfd079 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1187,7 +1187,6 @@ packages: '@ls-lint/ls-lint@2.2.3': resolution: {integrity: sha512-ekM12jNm/7O2I/hsRv9HvYkRdfrHpiV1epVuI2NP+eTIcEgdIdKkKCs9KgQydu/8R5YXTov9aHdOgplmCHLupw==} - cpu: [x64, arm64, s390x] os: [darwin, linux, win32] hasBin: true From 6caecbd238cb976464251b7ec5736f82a45b8ba1 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 19:40:06 +0530 Subject: [PATCH 02/37] validation --- lib/config/validation.ts | 1 + .../manager/custom/jsonata/index.spec.ts | 12 ++++ lib/modules/manager/custom/jsonata/index.ts | 58 ++++++++++++------- lib/modules/manager/custom/jsonata/readme.md | 13 +++-- lib/modules/manager/custom/jsonata/types.ts | 1 - lib/modules/manager/types.ts | 1 + 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/lib/config/validation.ts b/lib/config/validation.ts index d139cb42c10436..f9bb861dea63d0 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -530,6 +530,7 @@ export async function validateConfig( 'customType', 'description', 'fileMatch', + 'matchQueries', 'matchStrings', 'matchStringsStrategy', 'depNameTemplate', diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 8da9bc930bc6c7..f65f8b6d7f3c9a 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -245,6 +245,18 @@ describe('modules/manager/custom/jsonata/index', () => { expect(res?.deps).toHaveLength(2); }); + it('excludes and warns if invalid jsonata query found', async () => { + const config = { + matchQueries: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], + }; + const res = await extractPackageFile('{}', 'unused', config); + expect(res?.deps).toHaveLength(2); + expect(logger.warn).toHaveBeenCalledWith( + { err: expect.any(Object) }, + `Failed to compile JSONata query: {. Excluding it from queries.`, + ); + }); + it('extracts dependency with autoReplaceStringTemplate', async () => { const config = { matchQueries: [`{"depName": "foo"}`], diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index a627efb00e8588..90acf14eafb8bb 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -1,7 +1,7 @@ import is from '@sindresorhus/is'; import jsonata from 'jsonata'; import { logger } from '../../../../logger'; -import type { PackageDependency, PackageFile, Result } from '../../types'; +import type { PackageDependency, PackageFileContent } from '../../types'; import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; import { createDependency } from './utils'; @@ -11,14 +11,6 @@ export const defaultConfig = { fileMatch: [], }; -export interface CustomPackageFile extends PackageFile { - matchQueries: string[]; -} - -export function testJsonQuery(query: string): void { - jsonata(query); -} - const validMatchFields = [ 'depName', 'packageName', @@ -31,24 +23,48 @@ const validMatchFields = [ 'depType', ]; -function handleMatching( +async function handleMatching( json: unknown, packageFile: string, config: JsonataExtractConfig, -): PackageDependency[] { - return config.matchQueries - .flatMap((matchQuery) => jsonata(matchQuery).evaluate(json) ?? []) - .map((queryResult) => - createDependency(queryResult as Record, config), - ) - .filter(is.truthy); +): Promise { + // Pre-compile all JSONata expressions once + const compiledExpressions = config.matchQueries + .map((query) => { + try { + return jsonata(query); + } catch (err) { + logger.warn( + { err }, + `Failed to compile JSONata query: ${query}. Excluding it from queries.`, + ); + return null; + } + }) + .filter((expr) => expr !== null); + + // Execute all expressions in parallel + const results = await Promise.all( + compiledExpressions.map(async (expr) => { + const result = (await expr.evaluate(json)) ?? []; + return is.array(result) ? result : [result]; + }), + ); + + // Flatten results and create dependencies + return results + .flat() + .map((queryResult) => { + return createDependency(queryResult as Record, config); + }) + .filter((dep) => dep !== null); } -export function extractPackageFile( +export async function extractPackageFile( content: string, packageFile: string, config: JsonataExtractConfig, -): Result { +): Promise { let deps: PackageDependency[]; let json; @@ -62,12 +78,12 @@ export function extractPackageFile( return null; } - deps = handleMatching(json, packageFile, config); + deps = await handleMatching(json, packageFile, config); // filter all null values deps = deps.filter(is.truthy); if (deps.length) { - const res: CustomPackageFile & JSONataManagerTemplates = { + const res: PackageFileContent & JSONataManagerTemplates = { deps, matchQueries: config.matchQueries, }; diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index e05956222ecb95..744c3fe8ee66cd 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -1,17 +1,18 @@ -The `json-jsonata` manager is designed to allow users to manually configure Renovate for how to find dependencies in JSON files that aren't detected by the built-in package managers. +The `jsonata` manager is designed to allow users to manually configure Renovate for how to find dependencies in JSON files that aren't detected by the built-in package managers. This manager is unique in Renovate in that: - It is configurable via [JSONata](https://jsonata.org/) queries. -- Through the use of the `jsonataManagers` config, multiple "JSONata managers" can be created for the same repository. +- Through the use of the `customManagers` config, multiple "JSONata managers" can be created for the same repository. - It can extract any `datasource`. To configure it, use the following syntax: -``` +```javascript { - "jsonataManagers": [ + "customManagers": [ { + "type": "jsonata", "fileMatch": [""], "matchQueries": [''], ... @@ -36,9 +37,9 @@ Where `` is a [JSONata](https://docs.jsonata.org/overview.html) query tha } ``` -The meaning of each field is the same as the meaning of the capturing groups for regex managers. +The meaning of each field is the same as the meaning of the capturing groups for the `regex` manager. -The following configuration is also available for each `jsonManager` element, again with the same meaning as for the regex manager: +The following configuration is also available for each `jsonata` manager's element, again with the same meaning as for the regex manager: - `depNameTemplate`. - `packageNameTemplate`. diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts index 04832a4e6433b9..2fd06b24189d38 100644 --- a/lib/modules/manager/custom/jsonata/types.ts +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -13,7 +13,6 @@ export interface JSONataManagerTemplates { } export interface JSONataManagerConfig extends JSONataManagerTemplates { - // fileMatch: string[]; matchQueries: string[]; autoReplaceStringTemplate?: string; } diff --git a/lib/modules/manager/types.ts b/lib/modules/manager/types.ts index a2ccd07d47950d..63b859e1b2d5ed 100644 --- a/lib/modules/manager/types.ts +++ b/lib/modules/manager/types.ts @@ -64,6 +64,7 @@ export interface PackageFileContent> packageFileVersion?: string; skipInstalls?: boolean | null; matchStrings?: string[]; + matchQueries?: string[]; matchStringsStrategy?: MatchStringsStrategy; } From 0432ece337f7ba426705a6da4891f3bf4d172665 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 19:51:57 +0530 Subject: [PATCH 03/37] fix tests --- lib/modules/manager/custom/jsonata/index.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index f65f8b6d7f3c9a..0ed7a7561cdf12 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -21,7 +21,7 @@ describe('modules/manager/custom/jsonata/index', () => { "data_source": "nuget", "versioning": "maven", "extract_version": "custom-extract-version", - "registry_url": "http://brr.brr", + "registry_url": "https://brr.brr", "dep_type": "dev" } ] @@ -83,7 +83,7 @@ describe('modules/manager/custom/jsonata/index', () => { "data_source": "nuget", "versioning": "maven", "extract_version": "custom-extract-version", - "registry_url": "http://brr.brr", + "registry_url": "https://brr.brr", "dep_type": "dev" }, { @@ -118,7 +118,7 @@ describe('modules/manager/custom/jsonata/index', () => { extractVersionTemplate: '{{#if extractVersion}}{{extractVersion}}{{else}}default-extract-version{{/if}}', registryUrlTemplate: - '{{#if registryUrl}}{{registryUrl}}{{else}}http://default.registry.url{{/if}}', + '{{#if registryUrl}}{{registryUrl}}{{else}}https://default.registry.url{{/if}}', depTypeTemplate: '{{#if depType}}{{depType}}{{else}}default-dep-type{{/if}}', }; @@ -148,7 +148,7 @@ describe('modules/manager/custom/jsonata/index', () => { ), ).toHaveLength(1); expect( - res?.deps.filter((dep) => dep.registryUrls?.includes('http://brr.brr/')), + res?.deps.filter((dep) => dep.registryUrls?.includes('https://brr.brr/')), ).toHaveLength(1); expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); @@ -177,7 +177,7 @@ describe('modules/manager/custom/jsonata/index', () => { ).toHaveLength(1); expect( res?.deps.filter((dep) => - dep.registryUrls?.includes('http://default.registry.url/'), + dep.registryUrls?.includes('https://default.registry.url/'), ), ).toHaveLength(1); expect( From 754324dc0cfa52423cb912d97d57250b988ff0d6 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 19:56:11 +0530 Subject: [PATCH 04/37] update docs --- docs/usage/configuration-options.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 9a00259cbb5ec0..9dea5fca74ce2b 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -797,6 +797,19 @@ It will be compiled using Handlebars and the regex `groups` result. If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. +## matchQueries + +Each `matchQueries` must be a valid regular expression, optionally with named capture groups. + +Example: + +````json +{ + "matchQueries": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] +} + ### matchStrings Each `matchStrings` must be a valid regular expression, optionally with named capture groups. @@ -809,7 +822,7 @@ Example: "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" ] } -``` +```` ### matchStringsStrategy From c497553de5508d27bd1c9742afc3669d52aa784b Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 20:44:05 +0530 Subject: [PATCH 05/37] fix tests --- lib/modules/manager/custom/jsonata/index.spec.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 0ed7a7561cdf12..88486a6088b779 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -21,7 +21,7 @@ describe('modules/manager/custom/jsonata/index', () => { "data_source": "nuget", "versioning": "maven", "extract_version": "custom-extract-version", - "registry_url": "https://brr.brr", + "registry_url": "https://registry.npmjs.org", "dep_type": "dev" } ] @@ -65,8 +65,12 @@ describe('modules/manager/custom/jsonata/index', () => { (dep) => dep.extractVersion === 'custom-extract-version', ), ).toHaveLength(1); + // eslint-disable-next-line + console.log(res?.deps); expect( - res?.deps.filter((dep) => dep.registryUrls?.includes('http://brr.brr/')), + res?.deps.filter((dep) => + dep.registryUrls?.includes('https://registry.npmjs.org/'), + ), ).toHaveLength(1); expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); }); @@ -83,7 +87,7 @@ describe('modules/manager/custom/jsonata/index', () => { "data_source": "nuget", "versioning": "maven", "extract_version": "custom-extract-version", - "registry_url": "https://brr.brr", + "registry_url": "https://registry.npmjs.org", "dep_type": "dev" }, { @@ -148,7 +152,9 @@ describe('modules/manager/custom/jsonata/index', () => { ), ).toHaveLength(1); expect( - res?.deps.filter((dep) => dep.registryUrls?.includes('https://brr.brr/')), + res?.deps.filter((dep) => + dep.registryUrls?.includes('https://registry.npmjs.org/'), + ), ).toHaveLength(1); expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); From f8a32ea115eff85c10d47bea002bf9e267b9eed9 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 20:50:25 +0530 Subject: [PATCH 06/37] fix lint issue --- docs/usage/configuration-options.md | 5 +++-- lib/modules/manager/custom/jsonata/readme.md | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 9dea5fca74ce2b..0848f39fb83da0 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -803,12 +803,13 @@ Each `matchQueries` must be a valid regular expression, optionally with named ca Example: -````json +```json { "matchQueries": [ "packages.{ \"depName\": package, \"currentValue\": version }" ] } +``` ### matchStrings @@ -822,7 +823,7 @@ Example: "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" ] } -```` +``` ### matchStringsStrategy diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index 744c3fe8ee66cd..3170bac7ef1e82 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -2,9 +2,9 @@ The `jsonata` manager is designed to allow users to manually configure Renovate This manager is unique in Renovate in that: -- It is configurable via [JSONata](https://jsonata.org/) queries. -- Through the use of the `customManagers` config, multiple "JSONata managers" can be created for the same repository. -- It can extract any `datasource`. +- It is configurable via [JSONata](https://jsonata.org/) queries +- Through the use of the `customManagers` config, multiple "JSONata managers" can be created for the same repository +- It can extract any `datasource` To configure it, use the following syntax: From a375c516677c447ab3b3638fe1565a336dedbe20 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 21:31:53 +0530 Subject: [PATCH 07/37] refactor --- .../manager/custom/jsonata/index.spec.ts | 17 ++--- lib/modules/manager/custom/jsonata/index.ts | 73 +++---------------- lib/modules/manager/custom/jsonata/utils.ts | 39 ++++++++++ 3 files changed, 56 insertions(+), 73 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 88486a6088b779..dfdf40e22af9b0 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -1,11 +1,11 @@ -import { logger } from '../../../../logger'; +import { logger } from '../../../../../test/util'; import type { JsonataExtractConfig } from './types'; import { defaultConfig, extractPackageFile } from '.'; describe('modules/manager/custom/jsonata/index', () => { it('has default config', () => { expect(defaultConfig).toEqual({ - fileMatch: [], + pinDigests: false, }); }); @@ -192,16 +192,15 @@ describe('modules/manager/custom/jsonata/index', () => { }); it('returns null when content is not json', async () => { - jest.mock('renovate/lib/logger'); const res = await extractPackageFile( 'not-json', 'foo-file', {} as JsonataExtractConfig, ); expect(res).toBeNull(); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), - `error parsing 'foo-file'`, + `Error parsing 'foo-file'`, ); }); @@ -216,28 +215,26 @@ describe('modules/manager/custom/jsonata/index', () => { }); it('returns null if invalid template', async () => { - jest.mock('renovate/lib/logger'); const config = { matchQueries: [`{"depName": "foo"}`], versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template }; const res = await extractPackageFile('{}', 'unused', config); expect(res).toBeNull(); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), 'Error compiling template for JSONata manager', ); }); it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { - jest.mock('renovate/lib/logger'); const config = { matchQueries: [`{"depName": "foo"}`], registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', }; const res = await extractPackageFile('{}', 'unused', config); expect(res).not.toBeNull(); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.logger.warn).toHaveBeenCalledWith( { value: 'this-is-not-a-valid-url-foo' }, 'Invalid json manager registryUrl', ); @@ -257,7 +254,7 @@ describe('modules/manager/custom/jsonata/index', () => { }; const res = await extractPackageFile('{}', 'unused', config); expect(res?.deps).toHaveLength(2); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.logger.warn).toHaveBeenCalledWith( { err: expect.any(Object) }, `Failed to compile JSONata query: {. Excluding it from queries.`, ); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 90acf14eafb8bb..10e9ebabab90fc 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -1,87 +1,34 @@ -import is from '@sindresorhus/is'; -import jsonata from 'jsonata'; +import type { Category } from '../../../../constants'; import { logger } from '../../../../logger'; -import type { PackageDependency, PackageFileContent } from '../../types'; +import type { PackageFileContent } from '../../types'; import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; -import { createDependency } from './utils'; +import { handleMatching, validMatchFields } from './utils'; -export const supportedDatasources: string[] = ['*']; +export const categories: Category[] = ['custom']; export const defaultConfig = { - fileMatch: [], + pinDigests: false, }; - -const validMatchFields = [ - 'depName', - 'packageName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', - 'depType', -]; - -async function handleMatching( - json: unknown, - packageFile: string, - config: JsonataExtractConfig, -): Promise { - // Pre-compile all JSONata expressions once - const compiledExpressions = config.matchQueries - .map((query) => { - try { - return jsonata(query); - } catch (err) { - logger.warn( - { err }, - `Failed to compile JSONata query: ${query}. Excluding it from queries.`, - ); - return null; - } - }) - .filter((expr) => expr !== null); - - // Execute all expressions in parallel - const results = await Promise.all( - compiledExpressions.map(async (expr) => { - const result = (await expr.evaluate(json)) ?? []; - return is.array(result) ? result : [result]; - }), - ); - - // Flatten results and create dependencies - return results - .flat() - .map((queryResult) => { - return createDependency(queryResult as Record, config); - }) - .filter((dep) => dep !== null); -} +export const supportedDatasources = ['*']; +export const displayName = 'Jsonata'; export async function extractPackageFile( content: string, packageFile: string, config: JsonataExtractConfig, ): Promise { - let deps: PackageDependency[]; - let json; try { json = JSON.parse(content); } catch (err) { logger.warn( - { err, content, fileName: packageFile }, - `error parsing '${packageFile}'`, + { err, fileName: packageFile }, + `Error parsing '${packageFile}'`, ); return null; } - deps = await handleMatching(json, packageFile, config); - - // filter all null values - deps = deps.filter(is.truthy); + const deps = await handleMatching(json, packageFile, config); if (deps.length) { const res: PackageFileContent & JSONataManagerTemplates = { deps, diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index ff0d93c658e01d..404b1358c9909b 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -1,4 +1,6 @@ import { URL } from 'url'; +import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; import { logger } from '../../../../logger'; import * as template from '../../../../util/template'; import type { PackageDependency } from '../../types'; @@ -18,6 +20,43 @@ export const validMatchFields = [ type ValidMatchFields = (typeof validMatchFields)[number]; +export async function handleMatching( + json: unknown, + packageFile: string, + config: JsonataExtractConfig, +): Promise { + // Pre-compile all JSONata expressions once + const compiledExpressions = config.matchQueries + .map((query) => { + try { + return jsonata(query); + } catch (err) { + logger.warn( + { err }, + `Failed to compile JSONata query: ${query}. Excluding it from queries.`, + ); + return null; + } + }) + .filter((expr) => expr !== null); + + // Execute all expressions in parallel + const results = await Promise.all( + compiledExpressions.map(async (expr) => { + const result = (await expr.evaluate(json)) ?? []; + return is.array(result) ? result : [result]; + }), + ); + + // Flatten results and create dependencies + return results + .flat() + .map((queryResult) => { + return createDependency(queryResult as Record, config); + }) + .filter((dep) => dep !== null); +} + export function createDependency( queryResult: Record, config: JsonataExtractConfig, From 0b54c6dfbebaa44820f6701bd436b185518e302e Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 21:37:03 +0530 Subject: [PATCH 08/37] fix ci issues --- docs/usage/configuration-options.md | 2 +- pnpm-lock.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 0848f39fb83da0..5ce54b500226bf 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -797,7 +797,7 @@ It will be compiled using Handlebars and the regex `groups` result. If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. -## matchQueries +### matchQueries Each `matchQueries` must be a valid regular expression, optionally with named capture groups. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6041dfa30d87e1..311495b8a7355c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1187,6 +1187,7 @@ packages: '@ls-lint/ls-lint@2.2.3': resolution: {integrity: sha512-ekM12jNm/7O2I/hsRv9HvYkRdfrHpiV1epVuI2NP+eTIcEgdIdKkKCs9KgQydu/8R5YXTov9aHdOgplmCHLupw==} + cpu: [x64, arm64, s390x] os: [darwin, linux, win32] hasBin: true From 4c14ac6ef6b05c0ff93a30e9fe2ecb007520fe88 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 28 Nov 2024 23:21:04 +0530 Subject: [PATCH 09/37] remove duplicate code --- docs/usage/configuration-options.md | 2 +- lib/config/options/index.ts | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 5ce54b500226bf..395c2e6b2bc826 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -799,7 +799,7 @@ It will be compiled using Handlebars and the regex `groups` result. ### matchQueries -Each `matchQueries` must be a valid regular expression, optionally with named capture groups. +Each `matchQueries` must be a valid regular expression(escaped). Example: diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 529300e500bcf0..680c093c36f875 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2738,21 +2738,10 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, - { - name: 'matchStrings', - description: - 'Regex capture rule to use. Valid only within a `customManagers` object.', - type: 'array', - subType: 'string', - format: 'regex', - parents: ['customManagers'], - cli: false, - env: false, - }, { name: 'matchQueries', description: - 'JSON query to use. Valid only within a `customManagers` object of type `jsonata`.', + 'JSONata query to use. Valid only within a `customManagers` object of type `jsonata`.', type: 'array', subType: 'string', parents: ['customManagers'], From caf4513ec2ecad8ac0524e0703cd3bcb266f583f Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 29 Nov 2024 00:53:28 +0530 Subject: [PATCH 10/37] add jsonata to customType allowed values --- lib/config/options/index.ts | 2 +- lib/modules/manager/custom/jsonata/index.spec.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 680c093c36f875..14ebac3ebd3d92 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2733,7 +2733,7 @@ const options: RenovateOptions[] = [ description: 'Custom manager to use. Valid only within a `customManagers` object.', type: 'string', - allowedValues: ['regex'], + allowedValues: ['regex', 'jsonata'], parents: ['customManagers'], cli: false, env: false, diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index dfdf40e22af9b0..9996426fadcd71 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -65,8 +65,6 @@ describe('modules/manager/custom/jsonata/index', () => { (dep) => dep.extractVersion === 'custom-extract-version', ), ).toHaveLength(1); - // eslint-disable-next-line - console.log(res?.deps); expect( res?.deps.filter((dep) => dep.registryUrls?.includes('https://registry.npmjs.org/'), From 55fc985e3e05572cd6c4a5d91a24f0c3a60227fc Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 29 Nov 2024 13:51:31 +0530 Subject: [PATCH 11/37] Apply Suggestions --- docs/usage/configuration-options.md | 23 ++++++++++------ lib/config/options/index.ts | 13 +-------- lib/config/validation.ts | 1 - .../manager/custom/jsonata/index.spec.ts | 27 ++++++++++++------- lib/modules/manager/custom/jsonata/index.ts | 11 +++++--- lib/modules/manager/custom/jsonata/readme.md | 2 +- lib/modules/manager/custom/jsonata/types.ts | 4 +-- lib/modules/manager/custom/jsonata/utils.ts | 2 +- 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 395c2e6b2bc826..3f32f20967fdec 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -797,27 +797,26 @@ It will be compiled using Handlebars and the regex `groups` result. If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. -### matchQueries - -Each `matchQueries` must be a valid regular expression(escaped). +Each `matchQueries` must be a valid jsonata query(escaped). Example: ```json { - "matchQueries": [ - "packages.{ \"depName\": package, \"currentValue\": version }" - ] + "matchQueries": [] } ``` ### matchStrings -Each `matchStrings` must be a valid regular expression, optionally with named capture groups. +Each `matchStrings` must be one of thes two: + +1. a valid regular expression, optionally with named capture groups +2. a valid [JSONata](https://docs.jsonata.org/overview.html) query(escaped) Example: -```json +```json title="matchStrings with a valid regular expression" { "matchStrings": [ "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" @@ -825,6 +824,14 @@ Example: } ``` +```json title="matchStrings with a valid jsonata query" +{ + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] +} +``` + ### matchStringsStrategy `matchStringsStrategy` controls behavior when multiple `matchStrings` values are provided. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 14ebac3ebd3d92..81d1a00eb83ab0 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2738,23 +2738,12 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, - { - name: 'matchQueries', - description: - 'JSONata query to use. Valid only within a `customManagers` object of type `jsonata`.', - type: 'array', - subType: 'string', - parents: ['customManagers'], - cli: false, - env: false, - }, { name: 'matchStrings', description: - 'Regex capture rule to use. Valid only within a `customManagers` object.', + 'Regex or jsonata query rule to use. Valid only within a `customManagers` object.', type: 'array', subType: 'string', - format: 'regex', parents: ['customManagers'], cli: false, env: false, diff --git a/lib/config/validation.ts b/lib/config/validation.ts index f9bb861dea63d0..d139cb42c10436 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -530,7 +530,6 @@ export async function validateConfig( 'customType', 'description', 'fileMatch', - 'matchQueries', 'matchStrings', 'matchStringsStrategy', 'depNameTemplate', diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 9996426fadcd71..f60fea670d9b6a 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -27,7 +27,7 @@ describe('modules/manager/custom/jsonata/index', () => { ] }`; const config = { - matchQueries: [ + matchStrings: [ `packages.{ "depName": dep_name, "packageName": package_name, @@ -92,7 +92,7 @@ describe('modules/manager/custom/jsonata/index', () => { }] }`; const config = { - matchQueries: [ + matchStrings: [ `packages.{ "depName": dep_name, "packageName": package_name, @@ -202,9 +202,18 @@ describe('modules/manager/custom/jsonata/index', () => { ); }); + it('returns null when no content', async () => { + const res = await extractPackageFile( + '', + 'foo-file', + {} as JsonataExtractConfig, + ); + expect(res).toBeNull(); + }); + it('returns null if no dependencies found', async () => { const config = { - matchQueries: [ + matchStrings: [ 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', ], }; @@ -214,7 +223,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('returns null if invalid template', async () => { const config = { - matchQueries: [`{"depName": "foo"}`], + matchStrings: [`{"depName": "foo"}`], versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template }; const res = await extractPackageFile('{}', 'unused', config); @@ -227,7 +236,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { const config = { - matchQueries: [`{"depName": "foo"}`], + matchStrings: [`{"depName": "foo"}`], registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', }; const res = await extractPackageFile('{}', 'unused', config); @@ -238,9 +247,9 @@ describe('modules/manager/custom/jsonata/index', () => { ); }); - it('extracts multiple dependencies with multiple matchQueries', async () => { + it('extracts multiple dependencies with multiple matchStrings', async () => { const config = { - matchQueries: [`{"depName": "foo"}`, `{"depName": "bar"}`], + matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`], }; const res = await extractPackageFile('{}', 'unused', config); expect(res?.deps).toHaveLength(2); @@ -248,7 +257,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('excludes and warns if invalid jsonata query found', async () => { const config = { - matchQueries: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], + matchStrings: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], }; const res = await extractPackageFile('{}', 'unused', config); expect(res?.deps).toHaveLength(2); @@ -260,7 +269,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('extracts dependency with autoReplaceStringTemplate', async () => { const config = { - matchQueries: [`{"depName": "foo"}`], + matchStrings: [`{"depName": "foo"}`], autoReplaceStringTemplate: 'auto-replace-string-template', }; const res = await extractPackageFile('{}', 'values.yaml', config); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 10e9ebabab90fc..9c0c51c1b8f8c8 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -1,5 +1,6 @@ import type { Category } from '../../../../constants'; import { logger } from '../../../../logger'; +import { parseJson } from '../../../../util/common'; import type { PackageFileContent } from '../../types'; import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; import { handleMatching, validMatchFields } from './utils'; @@ -10,7 +11,7 @@ export const defaultConfig = { pinDigests: false, }; export const supportedDatasources = ['*']; -export const displayName = 'Jsonata'; +export const displayName = 'JSONata'; export async function extractPackageFile( content: string, @@ -19,7 +20,7 @@ export async function extractPackageFile( ): Promise { let json; try { - json = JSON.parse(content); + json = parseJson(content, packageFile); } catch (err) { logger.warn( { err, fileName: packageFile }, @@ -28,11 +29,15 @@ export async function extractPackageFile( return null; } + if (!json) { + return null; + } + const deps = await handleMatching(json, packageFile, config); if (deps.length) { const res: PackageFileContent & JSONataManagerTemplates = { deps, - matchQueries: config.matchQueries, + matchStrings: config.matchStrings, }; // copy over templates for autoreplace for (const field of validMatchFields.map( diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index 3170bac7ef1e82..c2f7edcb245f34 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -14,7 +14,7 @@ To configure it, use the following syntax: { "type": "jsonata", "fileMatch": [""], - "matchQueries": [''], + "matchStrings": [''], ... } ] diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts index 2fd06b24189d38..eb1da9e30042c1 100644 --- a/lib/modules/manager/custom/jsonata/types.ts +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -13,7 +13,7 @@ export interface JSONataManagerTemplates { } export interface JSONataManagerConfig extends JSONataManagerTemplates { - matchQueries: string[]; + matchStrings: string[]; autoReplaceStringTemplate?: string; } @@ -21,5 +21,5 @@ export interface JsonataExtractConfig extends ExtractConfig, JSONataManagerTemplates { autoReplaceStringTemplate?: string; - matchQueries: string[]; + matchStrings: string[]; } diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index 404b1358c9909b..6a81d449de5a1d 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -26,7 +26,7 @@ export async function handleMatching( config: JsonataExtractConfig, ): Promise { // Pre-compile all JSONata expressions once - const compiledExpressions = config.matchQueries + const compiledExpressions = config.matchStrings .map((query) => { try { return jsonata(query); From 69f6745a25654b903e788b8be96088cf58f93cfc Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 29 Nov 2024 13:52:19 +0530 Subject: [PATCH 12/37] refactor: remove unused types --- lib/modules/manager/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/manager/types.ts b/lib/modules/manager/types.ts index 63b859e1b2d5ed..a2ccd07d47950d 100644 --- a/lib/modules/manager/types.ts +++ b/lib/modules/manager/types.ts @@ -64,7 +64,6 @@ export interface PackageFileContent> packageFileVersion?: string; skipInstalls?: boolean | null; matchStrings?: string[]; - matchQueries?: string[]; matchStringsStrategy?: MatchStringsStrategy; } From dabb62710c59bc1a6f5401cdae8c566f90e17125 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Fri, 29 Nov 2024 13:54:17 +0530 Subject: [PATCH 13/37] docs: refactor --- docs/usage/configuration-options.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 3f32f20967fdec..726478a789055c 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -797,22 +797,12 @@ It will be compiled using Handlebars and the regex `groups` result. If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. -Each `matchQueries` must be a valid jsonata query(escaped). - -Example: - -```json -{ - "matchQueries": [] -} -``` - ### matchStrings -Each `matchStrings` must be one of thes two: +Each `matchStrings` must be one of the two: 1. a valid regular expression, optionally with named capture groups -2. a valid [JSONata](https://docs.jsonata.org/overview.html) query(escaped) +2. a valid [JSONata](https://docs.jsonata.org/overview.html) query (escaped) Example: From 7f60365dca38636d338ba2561195a3e38aeb7fa7 Mon Sep 17 00:00:00 2001 From: RahulGautamSingh Date: Fri, 29 Nov 2024 14:01:47 +0530 Subject: [PATCH 14/37] Update docs/usage/configuration-options.md Co-authored-by: Rhys Arkins --- docs/usage/configuration-options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 726478a789055c..185a9584448d57 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -801,8 +801,8 @@ It will be compiled using Handlebars and the regex `groups` result. Each `matchStrings` must be one of the two: -1. a valid regular expression, optionally with named capture groups -2. a valid [JSONata](https://docs.jsonata.org/overview.html) query (escaped) +1. a valid regular expression, optionally with named capture groups (if using `customType=regex`) +2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using customType=json) Example: From bfd00e1092acef00ece24bc198232fc52d035f79 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sat, 30 Nov 2024 13:24:41 +0530 Subject: [PATCH 15/37] apply suggestions --- docs/usage/configuration-options.md | 17 +++++++- lib/config/options/index.ts | 4 +- .../manager/custom/jsonata/index.spec.ts | 4 +- lib/modules/manager/custom/jsonata/index.ts | 41 ++++++++++--------- lib/modules/manager/custom/jsonata/utils.ts | 2 +- lib/modules/manager/custom/types.ts | 2 +- 6 files changed, 42 insertions(+), 28 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 185a9584448d57..102ebd68bc5526 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -777,6 +777,19 @@ Example: } ``` +```json +{ + "customManagers": [ + { + "customType": "jsonata", + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### datasourceTemplate If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. @@ -802,7 +815,7 @@ It will be compiled using Handlebars and the regex `groups` result. Each `matchStrings` must be one of the two: 1. a valid regular expression, optionally with named capture groups (if using `customType=regex`) -2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using customType=json) +2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using `customType=json`) Example: @@ -814,7 +827,7 @@ Example: } ``` -```json title="matchStrings with a valid jsonata query" +```json title="matchStrings with a valid JSONata query" { "matchStrings": [ "packages.{ \"depName\": package, \"currentValue\": version }" diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 81d1a00eb83ab0..1285df402cbcb3 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2733,7 +2733,7 @@ const options: RenovateOptions[] = [ description: 'Custom manager to use. Valid only within a `customManagers` object.', type: 'string', - allowedValues: ['regex', 'jsonata'], + allowedValues: ['jsonata', 'regex'], parents: ['customManagers'], cli: false, env: false, @@ -2741,7 +2741,7 @@ const options: RenovateOptions[] = [ { name: 'matchStrings', description: - 'Regex or jsonata query rule to use. Valid only within a `customManagers` object.', + 'Regex pattern or JSONata query to use. Valid only within a `customManagers` object.', type: 'array', subType: 'string', parents: ['customManagers'], diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index f60fea670d9b6a..226e8b1969eba2 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -198,7 +198,7 @@ describe('modules/manager/custom/jsonata/index', () => { expect(res).toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), - `Error parsing 'foo-file'`, + 'File is not a valid JSON file.', ); }); @@ -243,7 +243,7 @@ describe('modules/manager/custom/jsonata/index', () => { expect(res).not.toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( { value: 'this-is-not-a-valid-url-foo' }, - 'Invalid json manager registryUrl', + 'Invalid JSONata manager registryUrl', ); }); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 9c0c51c1b8f8c8..c36d0d5905c9b5 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -1,3 +1,4 @@ +import is from '@sindresorhus/is'; import type { Category } from '../../../../constants'; import { logger } from '../../../../logger'; import { parseJson } from '../../../../util/common'; @@ -24,34 +25,34 @@ export async function extractPackageFile( } catch (err) { logger.warn( { err, fileName: packageFile }, - `Error parsing '${packageFile}'`, + 'File is not a valid JSON file.', ); return null; } - if (!json) { + if (is.nullOrUndefined(json)) { return null; } const deps = await handleMatching(json, packageFile, config); - if (deps.length) { - const res: PackageFileContent & JSONataManagerTemplates = { - deps, - matchStrings: config.matchStrings, - }; - // copy over templates for autoreplace - for (const field of validMatchFields.map( - (f) => `${f}Template` as keyof JSONataManagerTemplates, - )) { - if (config[field]) { - res[field] = config[field]; - } - } - if (config.autoReplaceStringTemplate) { - res.autoReplaceStringTemplate = config.autoReplaceStringTemplate; - } - return res; + if (!deps.length) { + return null; } - return null; + const res: PackageFileContent & JSONataManagerTemplates = { + deps, + matchStrings: config.matchStrings, + }; + // copy over templates for autoreplace + for (const field of validMatchFields.map( + (f) => `${f}Template` as keyof JSONataManagerTemplates, + )) { + if (config[field]) { + res[field] = config[field]; + } + } + if (config.autoReplaceStringTemplate) { + res.autoReplaceStringTemplate = config.autoReplaceStringTemplate; + } + return res; } diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index 6a81d449de5a1d..eef1548b290b61 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -71,7 +71,7 @@ export function createDependency( const url = new URL(value).toString(); dependency.registryUrls = [url]; } catch { - logger.warn({ value }, 'Invalid json manager registryUrl'); + logger.warn({ value }, 'Invalid JSONata manager registryUrl'); } break; default: diff --git a/lib/modules/manager/custom/types.ts b/lib/modules/manager/custom/types.ts index 6e7b2595cc66cb..c7edda609f9e35 100644 --- a/lib/modules/manager/custom/types.ts +++ b/lib/modules/manager/custom/types.ts @@ -5,7 +5,7 @@ export interface CustomExtractConfig extends Partial, Partial {} -export type CustomManagerName = 'regex' | 'jsonata'; +export type CustomManagerName = 'jsonata' | 'regex'; export interface CustomManager extends Partial { customType: CustomManagerName; From 292c91f15262c4a3b0ddfb60853b9c3160826cc2 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sat, 30 Nov 2024 13:25:04 +0530 Subject: [PATCH 16/37] docs: redo structure --- lib/modules/manager/custom/jsonata/readme.md | 119 ++++++++++++------- 1 file changed, 76 insertions(+), 43 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index c2f7edcb245f34..a907e0227737f4 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -1,10 +1,50 @@ -The `jsonata` manager is designed to allow users to manually configure Renovate for how to find dependencies in JSON files that aren't detected by the built-in package managers. +With `customManagers` using `JSONata` queries you can configure Renovate so it finds dependencies in JSON files, that are not detected by its other built-in package managers. -This manager is unique in Renovate in that: +Renovate uses the `jsonata` package to process the `json` file content. Read about the [jsonata query language](https://docs.jsonata.org/overview.html) in their readme. + +The JSONata manager is unique in Renovate in because: - It is configurable via [JSONata](https://jsonata.org/) queries -- Through the use of the `customManagers` config, multiple "JSONata managers" can be created for the same repository - It can extract any `datasource` +- By using the `customManagers` config, you can create multiple "JSONata managers" the same repository + +### Required Fields + +The first two required fields are `fileMatch` and `matchStrings`: + +- `fileMatch` works the same as any manager +- `matchStrings` is a `JSONata` custom manager concept and is used for configuring a jsonata queries + +#### Information that Renovate needs about the dependency + +Before Renovate can look up a dependency and decide about updates, it must have this info about each dependency: + +| Info type | Required | Notes | Docs | +| :--------------------------------------------------- | :------- | :-------------------------------------------------------- | :----------------------------------------------------------------------------- | +| Name of the dependency | Yes | | | +| `datasource` | Yes | Example datasources: npm, Docker, GitHub tags, and so on. | [Supported datasources](../../datasource/index.md#supported-datasources) | +| Version scheme to use. Defaults to `semver-coerced`. | Yes | You may set another version scheme, like `pep440`. | [Supported versioning schemes](../../versioning/index.md#supported-versioning) | + +#### Required fields to be present in the resulting structure returned by the jsonata query + +You must: + +- Capture the `currentValue` of the dependency +- Capture the `depName` or `packageName`. Or use a template field: `depNameTemplate` and `packageNameTemplate` +- Capture the `datasource`, or a use `datasourceTemplate` config field + +#### Optional fields you can include in the resulting structure + +You may use any of these items: + +- `depType`, or a use `depTypeTemplate` config field +- `versioning`, or a use `versioningTemplate` config field. If neither are present, Renovate defaults to `semver-coerced` +- `extractVersion`, or use an `extractVersionTemplate` config field +- `currentDigest` +- `registryUrl`, or a use `registryUrlTemplate` config field. If it's a valid URL, it will be converted to the `registryUrls` field as a single-length array +- `indentation`. It must be either empty, or whitespace only (otherwise `indentation` will be reset to an empty string) + +### Usage To configure it, use the following syntax: @@ -23,41 +63,28 @@ To configure it, use the following syntax: Where `` is a [JSONata](https://docs.jsonata.org/overview.html) query that transform the contents into a JSON object with the following schema: -```json +```json5 { - "depName": "", - "packageName": "", - "currentValue": "", - "currentDigest": "", - "datasource": "", - "versioning": "", - "extractVersion": "", - "registryUrl": "", - "depType": "" + depName: '', + packageName: '', // fallback to depName + currentValue: '', + currentDigest: '', // optional + datasource: '', + versioning: '', // optional + extractVersion: '', // optional + registryUrl: '', // optional + depType: '', // optional } ``` -The meaning of each field is the same as the meaning of the capturing groups for the `regex` manager. - -The following configuration is also available for each `jsonata` manager's element, again with the same meaning as for the regex manager: - -- `depNameTemplate`. -- `packageNameTemplate`. -- `currentValueTemplate`. -- `currentDigestTemplate`. -- `datasourceTemplate`. -- `versioningTemplate`. -- `extractVersionTemplate`. -- `registryUrlTemplate`. -- `depTypeTemplate`. - -### Example queries +To be effective with the JSONata manager, you should understand jsonata queries. But enough examples may compensate for lack of experience. -Below are some example queries for the generic JSON manager. You can also use the [JSONata test website](https://try.jsonata.org) to experiment with queries. +#### Example queries -_Dependencies spread in different nodes, and we want to limit the extraction to a particular node:_ +Below are some example queries for the generic JSON manager. +You can also use the [JSONata test website](https://try.jsonata.org) to experiment with queries. -```json +```json title="Dependencies spread in different nodes, and we want to limit the extraction to a particular node" { "production": [ { @@ -80,9 +107,7 @@ Query: production.{ "depName": package, "currentValue": version } ``` -_Dependencies spread in different nodes, and we want to extract all of them as if they were in the same node:_ - -```json +```json title="Dependencies spread in different nodes, and we want to extract all of them as if they were in the same node" { "production": [ { @@ -105,9 +130,7 @@ Query: *.{ "depName": package, "currentValue": version } ``` -_The dependency name is in a JSON node name and the version is in a child leaf to that node_: - -```json +```json title="The dependency name is in a JSON node name and the version is in a child leaf to that node" { "foo": { "version": "1.2.3" @@ -124,9 +147,7 @@ Query: $each(function($v, $n) { { "depName": $n, "currentValue": $v.version } }) ``` -_The name of the dependency and the version are both value nodes of the same parent node:_ - -```json +```json title="The name of the dependency and the version are both value nodes of the same parent node" { "packages": [ { @@ -147,9 +168,7 @@ Query: packages.{ "depName": package, "currentValue": version } ``` -_The name of the dependency and the version are in the same string:_ - -```json +```json title="The name of the dependency and the version are in the same string" { "packages": ["foo@1.2.3", "bar@4.5.6"] } @@ -160,3 +179,17 @@ Query: ``` $map($map(packages, function ($v) { $split($v, "@") }), function ($v) { { "depName": $v[0], "currentVersion": $v[1] } }) ``` + +```json title="JSONata manager config to extract deps from package.json file in the renovate repository" +{ + "customType": "jsonata", + "fileMatch": ["package.json"], + "matchStrings": [ + "$each(dependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"dependencies\"}})", + "$each(devDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"devDependencies\"}})", + "$each(optionalDependencies, function($v, $k) { {\"depName\":$k, \"currentValue\": $v, \"depType\": \"optionalDependencies\"}})", + "{ \"depName\": \"pnpm\", \"currentValue\": $substring(packageManager, 5), \"depType\": \"packageManager\"}" + ], + "datasourceTemplate": "npm" +} +``` From 1324d838939b3a50d6e43ff1621338f8aaf65b3c Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 5 Dec 2024 19:50:57 +0530 Subject: [PATCH 17/37] apply suggestions --- lib/modules/manager/custom/jsonata/index.spec.ts | 8 ++++---- lib/modules/manager/custom/jsonata/index.ts | 2 +- lib/modules/manager/custom/jsonata/utils.ts | 7 ++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 226e8b1969eba2..eec4d84d323f27 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -198,7 +198,7 @@ describe('modules/manager/custom/jsonata/index', () => { expect(res).toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), - 'File is not a valid JSON file.', + 'Invalid JSON file(parsing failed)', ); }); @@ -242,7 +242,7 @@ describe('modules/manager/custom/jsonata/index', () => { const res = await extractPackageFile('{}', 'unused', config); expect(res).not.toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( - { value: 'this-is-not-a-valid-url-foo' }, + { url: 'this-is-not-a-valid-url-foo' }, 'Invalid JSONata manager registryUrl', ); }); @@ -262,8 +262,8 @@ describe('modules/manager/custom/jsonata/index', () => { const res = await extractPackageFile('{}', 'unused', config); expect(res?.deps).toHaveLength(2); expect(logger.logger.warn).toHaveBeenCalledWith( - { err: expect.any(Object) }, - `Failed to compile JSONata query: {. Excluding it from queries.`, + { err: expect.any(Object), query: '{' }, + 'Failed to compile JSONata query', ); }); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index c36d0d5905c9b5..03bddf3978b715 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -25,7 +25,7 @@ export async function extractPackageFile( } catch (err) { logger.warn( { err, fileName: packageFile }, - 'File is not a valid JSON file.', + 'Invalid JSON file(parsing failed)', ); return null; } diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index eef1548b290b61..bd57859313126a 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -31,10 +31,7 @@ export async function handleMatching( try { return jsonata(query); } catch (err) { - logger.warn( - { err }, - `Failed to compile JSONata query: ${query}. Excluding it from queries.`, - ); + logger.warn({ query, err }, 'Failed to compile JSONata query'); return null; } }) @@ -71,7 +68,7 @@ export function createDependency( const url = new URL(value).toString(); dependency.registryUrls = [url]; } catch { - logger.warn({ value }, 'Invalid JSONata manager registryUrl'); + logger.warn({ url: value }, 'Invalid JSONata manager registryUrl'); } break; default: From ff13a219bf55bbe9e79604800a584fb0d312b8fb Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 8 Dec 2024 01:09:55 +0530 Subject: [PATCH 18/37] Apply Suggestions --- .../manager/custom/jsonata/index.spec.ts | 166 +++++++++--------- lib/modules/manager/custom/jsonata/index.ts | 2 +- lib/modules/manager/custom/jsonata/schema.ts | 16 ++ lib/modules/manager/custom/jsonata/utils.ts | 66 ++++--- 4 files changed, 139 insertions(+), 111 deletions(-) create mode 100644 lib/modules/manager/custom/jsonata/schema.ts diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index eec4d84d323f27..f6559c9208ad86 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { logger } from '../../../../../test/util'; import type { JsonataExtractConfig } from './types'; import { defaultConfig, extractPackageFile } from '.'; @@ -10,7 +11,7 @@ describe('modules/manager/custom/jsonata/index', () => { }); it('extracts data when no templates are used', async () => { - const json = ` + const json = codeBlock` { "packages": [ { @@ -44,37 +45,26 @@ describe('modules/manager/custom/jsonata/index', () => { const res = await extractPackageFile(json, 'unused', config); expect(res?.deps).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( - 1, - ); - expect( - res?.deps.filter((dep) => dep.currentValue === '1.2.3'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.currentDigest === '1234'), - ).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( - 1, - ); - expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( - 1, - ); - expect( - res?.deps.filter( - (dep) => dep.extractVersion === 'custom-extract-version', - ), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => - dep.registryUrls?.includes('https://registry.npmjs.org/'), - ), - ).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + packageName: 'fii', + currentValue: '1.2.3', + currentDigest: '1234', + datasource: 'nuget', + versioning: 'maven', + extractVersion: 'custom-extract-version', + registryUrls: ['https://registry.npmjs.org/'], + depType: 'dev', + }, + ], + matchStrings: config.matchStrings, + }); }); it('applies templates', async () => { - const json = ` + const json = codeBlock` { "packages": [ { @@ -127,66 +117,68 @@ describe('modules/manager/custom/jsonata/index', () => { const res = await extractPackageFile(json, 'unused', config); expect(res?.deps).toHaveLength(2); + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + packageName: 'fii', + currentValue: '1.2.3', + currentDigest: '1234', + datasource: 'nuget', + versioning: 'maven', + extractVersion: 'custom-extract-version', + registryUrls: ['https://registry.npmjs.org/'], + depType: 'dev', + }, + { + depName: 'default-dep-name', + packageName: 'default-package-name', + currentValue: 'default-current-value', + currentDigest: 'default-current-digest', + datasource: 'default-datasource', + versioning: 'default-versioning', + extractVersion: 'default-extract-version', + registryUrls: ['https://default.registry.url/'], + depType: 'default-dep-type', + }, + ], + matchStrings: config.matchStrings, + }); + }); - expect(res?.deps.filter((dep) => dep.depName === 'foo')).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.packageName === 'fii')).toHaveLength( - 1, - ); - expect( - res?.deps.filter((dep) => dep.currentValue === '1.2.3'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.currentDigest === '1234'), - ).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.datasource === 'nuget')).toHaveLength( - 1, - ); - expect(res?.deps.filter((dep) => dep.versioning === 'maven')).toHaveLength( - 1, - ); - expect( - res?.deps.filter( - (dep) => dep.extractVersion === 'custom-extract-version', - ), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => - dep.registryUrls?.includes('https://registry.npmjs.org/'), - ), - ).toHaveLength(1); - expect(res?.deps.filter((dep) => dep.depType === 'dev')).toHaveLength(1); + it('logs warning if query result does not match schema', async () => { + const json = codeBlock` + { + "packages": [ + { + "dep_name": "foo", + "package_name": "fii", + "current_value": 1, + "current_digest": "1234", + "data_source": "nuget", + "versioning": "maven", + "extract_version": "custom-extract-version", + "registry_url": "https://registry.npmjs.org", + "dep_type": "dev" + } + ] + }`; + const config = { + matchStrings: [ + `packages.{ + "depName": dep_name, + "currentValue": current_value, + "datasource": data_source + }`, + ], + }; + const res = await extractPackageFile(json, 'unused', config); - expect( - res?.deps.filter((dep) => dep.depName === 'default-dep-name'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.packageName === 'default-package-name'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.currentValue === 'default-current-value'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.currentDigest === 'default-current-digest'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.datasource === 'default-datasource'), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.versioning === 'default-versioning'), - ).toHaveLength(1); - expect( - res?.deps.filter( - (dep) => dep.extractVersion === 'default-extract-version', - ), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => - dep.registryUrls?.includes('https://default.registry.url/'), - ), - ).toHaveLength(1); - expect( - res?.deps.filter((dep) => dep.depType === 'default-dep-type'), - ).toHaveLength(1); + expect(res).toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'Error while parsing dep info', + ); }); it('returns null when content is not json', async () => { diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 03bddf3978b715..621cea08d5c499 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -19,7 +19,7 @@ export async function extractPackageFile( packageFile: string, config: JsonataExtractConfig, ): Promise { - let json; + let json: unknown; try { json = parseJson(content, packageFile); } catch (err) { diff --git a/lib/modules/manager/custom/jsonata/schema.ts b/lib/modules/manager/custom/jsonata/schema.ts new file mode 100644 index 00000000000000..e6fc2027e683ac --- /dev/null +++ b/lib/modules/manager/custom/jsonata/schema.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +const DepObjectSchema = z.object({ + currentValue: z.string().optional(), + datasource: z.string().optional(), + depName: z.string().optional(), + packageName: z.string().optional(), + currentDigest: z.string().optional(), + versioning: z.string().optional(), + depType: z.string().optional(), + registryUrl: z.string().optional(), + extractVersion: z.string().optional(), + indentation: z.number().optional(), +}); + +export const QueryResultZodSchema = z.array(DepObjectSchema); diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index bd57859313126a..6357d311976380 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -1,9 +1,10 @@ -import { URL } from 'url'; import is from '@sindresorhus/is'; import jsonata from 'jsonata'; import { logger } from '../../../../logger'; import * as template from '../../../../util/template'; +import { parseUrl } from '../../../../util/url'; import type { PackageDependency } from '../../types'; +import { QueryResultZodSchema } from './schema'; import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; export const validMatchFields = [ @@ -40,8 +41,21 @@ export async function handleMatching( // Execute all expressions in parallel const results = await Promise.all( compiledExpressions.map(async (expr) => { - const result = (await expr.evaluate(json)) ?? []; - return is.array(result) ? result : [result]; + try { + // can either be a single object, an array of objects or undefined (no match) + let result = (await expr.evaluate(json)) ?? []; + if (is.emptyObject(result) || is.emptyArray(result)) { + return []; + } + + result = is.array(result) ? result : [result]; + + QueryResultZodSchema.parse(result); + return structuredClone(result); + } catch (err) { + logger.warn({ err }, 'Error while parsing dep info'); + return []; + } }), ); @@ -49,7 +63,7 @@ export async function handleMatching( return results .flat() .map((queryResult) => { - return createDependency(queryResult as Record, config); + return createDependency(queryResult, config); }) .filter((dep) => dep !== null); } @@ -60,30 +74,13 @@ export function createDependency( ): PackageDependency | null { const dependency: PackageDependency = {}; - function updateDependency(field: ValidMatchFields, value: string): void { - switch (field) { - case 'registryUrl': - // check if URL is valid and pack inside an array - try { - const url = new URL(value).toString(); - dependency.registryUrls = [url]; - } catch { - logger.warn({ url: value }, 'Invalid JSONata manager registryUrl'); - } - break; - default: - dependency[field] = value; - break; - } - } - for (const field of validMatchFields) { const fieldTemplate = `${field}Template` as keyof JSONataManagerTemplates; const tmpl = config[fieldTemplate]; if (tmpl) { try { const compiled = template.compile(tmpl, queryResult, false); - updateDependency(field, compiled); + updateDependency(field, compiled, dependency); } catch { logger.warn( { template: tmpl }, @@ -92,8 +89,31 @@ export function createDependency( return null; } } else if (queryResult[field]) { - updateDependency(field, queryResult[field]); + updateDependency(field, queryResult[field], dependency); + } + } + return dependency; +} + +function updateDependency( + field: ValidMatchFields, + value: string, + dependency: PackageDependency, +): PackageDependency { + switch (field) { + case 'registryUrl': { + const url = parseUrl(value)?.toString(); + if (!url) { + logger.warn({ url: value }, 'Invalid JSONata manager registryUrl'); + break; + } + dependency.registryUrls = [url]; + break; } + default: + dependency[field] = value; + break; } + return dependency; } From c8db81516c3eaeb374fc342c14bf29ae4ce69374 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 8 Dec 2024 01:12:16 +0530 Subject: [PATCH 19/37] docs: remove redundant codeblock --- lib/modules/manager/custom/jsonata/readme.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index a907e0227737f4..5eb836cdf543ea 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -63,20 +63,6 @@ To configure it, use the following syntax: Where `` is a [JSONata](https://docs.jsonata.org/overview.html) query that transform the contents into a JSON object with the following schema: -```json5 -{ - depName: '', - packageName: '', // fallback to depName - currentValue: '', - currentDigest: '', // optional - datasource: '', - versioning: '', // optional - extractVersion: '', // optional - registryUrl: '', // optional - depType: '', // optional -} -``` - To be effective with the JSONata manager, you should understand jsonata queries. But enough examples may compensate for lack of experience. #### Example queries From c92ad2b37d91a1468450ae198669883fe1453f40 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sun, 8 Dec 2024 01:14:23 +0530 Subject: [PATCH 20/37] refactor: tests --- lib/modules/manager/custom/jsonata/index.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index f6559c9208ad86..4db24be5a5f83f 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -44,7 +44,6 @@ describe('modules/manager/custom/jsonata/index', () => { }; const res = await extractPackageFile(json, 'unused', config); - expect(res?.deps).toHaveLength(1); expect(res).toMatchObject({ deps: [ { @@ -116,7 +115,6 @@ describe('modules/manager/custom/jsonata/index', () => { }; const res = await extractPackageFile(json, 'unused', config); - expect(res?.deps).toHaveLength(2); expect(res).toMatchObject({ deps: [ { From 28e64e51634c7bc128e1b554fa4a297fb9189491 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:20:08 +0530 Subject: [PATCH 21/37] fix: docs --- docs/usage/configuration-options.md | 19 +++++++++++++++---- lib/modules/manager/custom/jsonata/readme.md | 3 ++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 102ebd68bc5526..a23bf8e8d479f4 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -698,22 +698,22 @@ You can define custom managers to handle: - Proprietary file formats or conventions - Popular file formats not yet supported as a manager by Renovate -Currently we only have one custom manager. +Currently we only have two custom managers. The `regex` manager which is based on using Regular Expression named capture groups. +The `jsonata` manager which is based on using JSONata queries. -You must have a named capture group matching (e.g. `(?.*)`) _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: +You must have capture/extract the following three fields _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: - `datasource` - `depName` and / or `packageName` - `currentValue` -Use named capture group matching _or_ set a corresponding template. We recommend you use only _one_ of these methods, or you'll get confused. We recommend that you also tell Renovate what `versioning` to use. If the `versioning` field is missing, then Renovate defaults to using `semver` versioning. -For more details and examples about it, see our [documentation for the `regex` manager](modules/manager/regex/index.md). +For more details and examples about it, see our documentation for the [`regex` manager](modules/manager/regex/index.md) and the [`JSONata` manager](modules/manager/jsonata/index.md). For template fields, use the triple brace `{{{ }}}` notation to avoid Handlebars escaping any special characters. @@ -755,6 +755,10 @@ This will lead to following update where `1.21-alpine` is the newest version of image: my.new.registry/aRepository/andImage:1.21-alpine ``` + +!!! note + Can only be used with the custom regex maanger. + ### currentValueTemplate If the `currentValue` for a dependency is not captured with a named group then it can be defined in config using this field. @@ -769,6 +773,7 @@ Example: "customManagers": [ { "customType": "regex", + "fileMatch": ["Dockerfile"], "matchStrings": [ "ENV .*?_VERSION=(?.*) # (?.*?)/(?.*?)\\s" ] @@ -782,6 +787,8 @@ Example: "customManagers": [ { "customType": "jsonata", + "fileFormat": "json", + "fileMatch": ["file.json"], "matchStrings": [ "packages.{ \"depName\": package, \"currentValue\": version }" ] @@ -844,6 +851,10 @@ Three options are available: - `recursive` - `combination` + +!!! note + Only to be used with custom regex manager. + #### any Each provided `matchString` will be matched individually to the content of the `packageFile`. diff --git a/lib/modules/manager/custom/jsonata/readme.md b/lib/modules/manager/custom/jsonata/readme.md index 5eb836cdf543ea..2490ce58979723 100644 --- a/lib/modules/manager/custom/jsonata/readme.md +++ b/lib/modules/manager/custom/jsonata/readme.md @@ -52,7 +52,8 @@ To configure it, use the following syntax: { "customManagers": [ { - "type": "jsonata", + "customType": "jsonata", + "fileFormat": "json", "fileMatch": [""], "matchStrings": [''], ... From cb255575521ef288b1e20e2ab36b7fe77892a94e Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:20:27 +0530 Subject: [PATCH 22/37] fix: types --- lib/modules/manager/custom/types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/custom/types.ts b/lib/modules/manager/custom/types.ts index c7edda609f9e35..a6aa1ff22ed825 100644 --- a/lib/modules/manager/custom/types.ts +++ b/lib/modules/manager/custom/types.ts @@ -7,7 +7,9 @@ export interface CustomExtractConfig export type CustomManagerName = 'jsonata' | 'regex'; -export interface CustomManager extends Partial { +export interface CustomManager + extends Partial, + Partial { customType: CustomManagerName; fileMatch: string[]; } From d4cfb2da7ef19c61b871f8a5a989b7bd49470e27 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:21:21 +0530 Subject: [PATCH 23/37] feat: add new field fileFormat --- .../manager/custom/jsonata/index.spec.ts | 28 +++++++++---------- lib/modules/manager/custom/jsonata/index.ts | 28 ++++++++----------- lib/modules/manager/custom/jsonata/types.ts | 4 +-- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 4db24be5a5f83f..2f97272ae2776f 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -28,6 +28,7 @@ describe('modules/manager/custom/jsonata/index', () => { ] }`; const config = { + fileFormat: 'json', matchStrings: [ `packages.{ "depName": dep_name, @@ -81,6 +82,7 @@ describe('modules/manager/custom/jsonata/index', () => { }] }`; const config = { + fileFormat: 'json', matchStrings: [ `packages.{ "depName": dep_name, @@ -162,6 +164,7 @@ describe('modules/manager/custom/jsonata/index', () => { ] }`; const config = { + fileFormat: 'json', matchStrings: [ `packages.{ "depName": dep_name, @@ -193,16 +196,18 @@ describe('modules/manager/custom/jsonata/index', () => { }); it('returns null when no content', async () => { - const res = await extractPackageFile( - '', - 'foo-file', - {} as JsonataExtractConfig, - ); + const res = await extractPackageFile('', 'foo-file', { + fileFormat: 'json', + matchStrings: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + } as JsonataExtractConfig); expect(res).toBeNull(); }); it('returns null if no dependencies found', async () => { const config = { + fileFormat: 'json', matchStrings: [ 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', ], @@ -213,6 +218,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('returns null if invalid template', async () => { const config = { + fileFormat: 'json', matchStrings: [`{"depName": "foo"}`], versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template }; @@ -226,6 +232,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { const config = { + fileFormat: 'json', matchStrings: [`{"depName": "foo"}`], registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', }; @@ -239,6 +246,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('extracts multiple dependencies with multiple matchStrings', async () => { const config = { + fileFormat: 'json', matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`], }; const res = await extractPackageFile('{}', 'unused', config); @@ -247,6 +255,7 @@ describe('modules/manager/custom/jsonata/index', () => { it('excludes and warns if invalid jsonata query found', async () => { const config = { + fileFormat: 'json', matchStrings: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], }; const res = await extractPackageFile('{}', 'unused', config); @@ -256,13 +265,4 @@ describe('modules/manager/custom/jsonata/index', () => { 'Failed to compile JSONata query', ); }); - - it('extracts dependency with autoReplaceStringTemplate', async () => { - const config = { - matchStrings: [`{"depName": "foo"}`], - autoReplaceStringTemplate: 'auto-replace-string-template', - }; - const res = await extractPackageFile('{}', 'values.yaml', config); - expect(res?.autoReplaceStringTemplate).toBe('auto-replace-string-template'); - }); }); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 621cea08d5c499..98ab5b67ab56d0 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -3,8 +3,8 @@ import type { Category } from '../../../../constants'; import { logger } from '../../../../logger'; import { parseJson } from '../../../../util/common'; import type { PackageFileContent } from '../../types'; -import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; -import { handleMatching, validMatchFields } from './utils'; +import type { JsonataExtractConfig } from './types'; +import { handleMatching } from './utils'; export const categories: Category[] = ['custom']; @@ -21,7 +21,15 @@ export async function extractPackageFile( ): Promise { let json: unknown; try { - json = parseJson(content, packageFile); + switch (config.fileFormat) { + case 'json': + json = parseJson(content, packageFile); + break; + default: + throw new Error( + 'Invalid file format. JSONata only supports json files currently.', + ); + } } catch (err) { logger.warn( { err, fileName: packageFile }, @@ -39,20 +47,8 @@ export async function extractPackageFile( return null; } - const res: PackageFileContent & JSONataManagerTemplates = { + return { deps, matchStrings: config.matchStrings, }; - // copy over templates for autoreplace - for (const field of validMatchFields.map( - (f) => `${f}Template` as keyof JSONataManagerTemplates, - )) { - if (config[field]) { - res[field] = config[field]; - } - } - if (config.autoReplaceStringTemplate) { - res.autoReplaceStringTemplate = config.autoReplaceStringTemplate; - } - return res; } diff --git a/lib/modules/manager/custom/jsonata/types.ts b/lib/modules/manager/custom/jsonata/types.ts index eb1da9e30042c1..70852a81bfc41d 100644 --- a/lib/modules/manager/custom/jsonata/types.ts +++ b/lib/modules/manager/custom/jsonata/types.ts @@ -13,13 +13,13 @@ export interface JSONataManagerTemplates { } export interface JSONataManagerConfig extends JSONataManagerTemplates { + fileFormat: string; matchStrings: string[]; - autoReplaceStringTemplate?: string; } export interface JsonataExtractConfig extends ExtractConfig, JSONataManagerTemplates { - autoReplaceStringTemplate?: string; + fileFormat: string; matchStrings: string[]; } From 96c00243d1ae29f1f84a9e5c3f1b8238a000c1a9 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:27:10 +0530 Subject: [PATCH 24/37] refactor: simplify logic for handleMatching() --- lib/modules/manager/custom/jsonata/utils.ts | 85 +++++++++------------ 1 file changed, 35 insertions(+), 50 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index 6357d311976380..b31c65e52d1906 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -4,68 +4,53 @@ import { logger } from '../../../../logger'; import * as template from '../../../../util/template'; import { parseUrl } from '../../../../util/url'; import type { PackageDependency } from '../../types'; +import type { ValidMatchFields } from '../utils'; +import { checkIsValidDependency, validMatchFields } from '../utils'; import { QueryResultZodSchema } from './schema'; import type { JSONataManagerTemplates, JsonataExtractConfig } from './types'; -export const validMatchFields = [ - 'depName', - 'packageName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', - 'depType', -] as const; - -type ValidMatchFields = (typeof validMatchFields)[number]; - export async function handleMatching( json: unknown, packageFile: string, config: JsonataExtractConfig, ): Promise { - // Pre-compile all JSONata expressions once - const compiledExpressions = config.matchStrings - .map((query) => { - try { - return jsonata(query); - } catch (err) { - logger.warn({ query, err }, 'Failed to compile JSONata query'); - return null; - } - }) - .filter((expr) => expr !== null); + const results: Record[] = []; + const { matchStrings: jsonataQueries = [] } = config; + for (const query of jsonataQueries) { + // won't fail as this is verified during config validation + const jsonataExpression = jsonata(query); + // this does not throw error, just returns undefined if no matches + let queryResult = (await jsonataExpression.evaluate(json)) ?? []; - // Execute all expressions in parallel - const results = await Promise.all( - compiledExpressions.map(async (expr) => { - try { - // can either be a single object, an array of objects or undefined (no match) - let result = (await expr.evaluate(json)) ?? []; - if (is.emptyObject(result) || is.emptyArray(result)) { - return []; - } - - result = is.array(result) ? result : [result]; + if (is.emptyObject(queryResult) || is.emptyArray(queryResult)) { + logger.warn( + { + jsonataQuery: query, + packageFile, + }, + 'The jsonata query returned no matches. Possible error, please check your query', + ); + continue; + } - QueryResultZodSchema.parse(result); - return structuredClone(result); - } catch (err) { - logger.warn({ err }, 'Error while parsing dep info'); - return []; - } - }), - ); + queryResult = is.array(queryResult) ? queryResult : [queryResult]; + const parsed = QueryResultZodSchema.safeParse(queryResult); + if (parsed.success) { + results.concat(structuredClone(parsed.data)); + } else { + logger.warn( + { err: parsed.error, jsonataQuery: query, packageFile, queryResult }, + 'Query results failed schema validation', + ); + } + } - // Flatten results and create dependencies return results - .flat() - .map((queryResult) => { - return createDependency(queryResult, config); - }) - .filter((dep) => dep !== null); + .map((dep) => createDependency(dep, config)) + .filter(is.truthy) + .filter((dep) => + checkIsValidDependency(dep, packageFile, 'custom.jsonata'), + ); } export function createDependency( From 1ef876475395ac1c27c013f3959cfb8bcab9384a Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:28:09 +0530 Subject: [PATCH 25/37] refactor: apply DRY concept --- lib/modules/manager/custom/regex/index.ts | 2 +- .../manager/custom/regex/strategies.ts | 29 ++-------- lib/modules/manager/custom/regex/utils.ts | 32 +---------- lib/modules/manager/custom/utils.ts | 57 +++++++++++++++++++ 4 files changed, 64 insertions(+), 56 deletions(-) create mode 100644 lib/modules/manager/custom/utils.ts diff --git a/lib/modules/manager/custom/regex/index.ts b/lib/modules/manager/custom/regex/index.ts index 10ddca3d0dd5d5..87dc8e697d02cf 100644 --- a/lib/modules/manager/custom/regex/index.ts +++ b/lib/modules/manager/custom/regex/index.ts @@ -6,9 +6,9 @@ import type { PackageFileContent, Result, } from '../../types'; +import { validMatchFields } from '../utils'; import { handleAny, handleCombination, handleRecursive } from './strategies'; import type { RegexManagerConfig, RegexManagerTemplates } from './types'; -import { validMatchFields } from './utils'; export const categories: Category[] = ['custom']; diff --git a/lib/modules/manager/custom/regex/strategies.ts b/lib/modules/manager/custom/regex/strategies.ts index 2e489e43bf0598..4f36fa4395dc42 100644 --- a/lib/modules/manager/custom/regex/strategies.ts +++ b/lib/modules/manager/custom/regex/strategies.ts @@ -1,11 +1,10 @@ import is from '@sindresorhus/is'; -import { logger } from '../../../../logger'; import { regEx } from '../../../../util/regex'; import type { PackageDependency } from '../../types'; +import { checkIsValidDependency } from '../utils'; import type { RecursionParameter, RegexManagerConfig } from './types'; import { createDependency, - isValidDependency, mergeExtractionTemplate, mergeGroups, regexMatchAll, @@ -32,7 +31,7 @@ export function handleAny( ) .filter(is.truthy) .filter((dep: PackageDependency) => - checkIsValidDependency(dep, packageFile), + checkIsValidDependency(dep, packageFile, 'custom.regex'), ); } @@ -61,7 +60,7 @@ export function handleCombination( return [createDependency(extraction, config)] .filter(is.truthy) .filter((dep: PackageDependency) => - checkIsValidDependency(dep, packageFile), + checkIsValidDependency(dep, packageFile, 'custom.regex'), ); } @@ -84,7 +83,7 @@ export function handleRecursive( }) .filter(is.truthy) .filter((dep: PackageDependency) => - checkIsValidDependency(dep, packageFile), + checkIsValidDependency(dep, packageFile, 'custom.regex'), ); } @@ -116,23 +115,3 @@ function processRecursive(parameters: RecursionParameter): PackageDependency[] { }); }); } - -function checkIsValidDependency( - dep: PackageDependency, - packageFile: string, -): boolean { - const isValid = isValidDependency(dep); - if (!isValid) { - const meta = { - packageDependency: dep, - packageFile, - }; - logger.trace( - meta, - 'Discovered a package dependency by matching regex, but it did not pass validation. Discarding', - ); - return isValid; - } - - return isValid; -} diff --git a/lib/modules/manager/custom/regex/utils.ts b/lib/modules/manager/custom/regex/utils.ts index 350f639d9da434..3714b1f42deafd 100644 --- a/lib/modules/manager/custom/regex/utils.ts +++ b/lib/modules/manager/custom/regex/utils.ts @@ -4,27 +4,14 @@ import { migrateDatasource } from '../../../../config/migrations/custom/datasour import { logger } from '../../../../logger'; import * as template from '../../../../util/template'; import type { PackageDependency } from '../../types'; +import type { ValidMatchFields } from '../utils'; +import { validMatchFields } from '../utils'; import type { ExtractionTemplate, RegexManagerConfig, RegexManagerTemplates, } from './types'; -export const validMatchFields = [ - 'depName', - 'packageName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', - 'depType', - 'indentation', -] as const; - -type ValidMatchFields = (typeof validMatchFields)[number]; - function updateDependency( dependency: PackageDependency, field: ValidMatchFields, @@ -119,18 +106,3 @@ export function mergeExtractionTemplate( replaceString: addition.replaceString ?? base.replaceString, }; } - -export function isValidDependency({ - depName, - currentValue, - currentDigest, - packageName, -}: PackageDependency): boolean { - // check if all the fields are set - return ( - (is.nonEmptyStringAndNotWhitespace(depName) || - is.nonEmptyStringAndNotWhitespace(packageName)) && - (is.nonEmptyStringAndNotWhitespace(currentDigest) || - is.nonEmptyStringAndNotWhitespace(currentValue)) - ); -} diff --git a/lib/modules/manager/custom/utils.ts b/lib/modules/manager/custom/utils.ts new file mode 100644 index 00000000000000..024ac0d078d4b3 --- /dev/null +++ b/lib/modules/manager/custom/utils.ts @@ -0,0 +1,57 @@ +import is from '@sindresorhus/is'; +import { logger } from '../../../logger'; +import type { PackageDependency } from '../types'; + +export const validMatchFields = [ + 'depName', + 'packageName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', + 'depType', + 'indentation', +] as const; + +export type ValidMatchFields = (typeof validMatchFields)[number]; + +export function isValidDependency({ + depName, + currentValue, + currentDigest, + packageName, + datasource, +}: PackageDependency): boolean { + // check if all the fields are set + return ( + (is.nonEmptyStringAndNotWhitespace(depName) || + is.nonEmptyStringAndNotWhitespace(packageName)) && + (is.nonEmptyStringAndNotWhitespace(currentDigest) || + is.nonEmptyStringAndNotWhitespace(currentValue)) && + is.nonEmptyStringAndNotWhitespace(datasource) + ); +} + +export function checkIsValidDependency( + dep: PackageDependency, + packageFile: string, + manager: string, +): boolean { + const isValid = isValidDependency(dep); + if (!isValid) { + const meta = { + packageDependency: dep, + packageFile, + manager, + }; + logger.trace( + meta, + 'Discovered a package dependency, but it did not pass validation. Discarding', + ); + return isValid; + } + + return isValid; +} From ffd62280bdfc7a6a3a88e9fe5a71c1cbe552145d Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 11 Dec 2024 21:28:28 +0530 Subject: [PATCH 26/37] fix(types): indentation --- lib/modules/manager/custom/jsonata/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/custom/jsonata/schema.ts b/lib/modules/manager/custom/jsonata/schema.ts index e6fc2027e683ac..8a06911695d91e 100644 --- a/lib/modules/manager/custom/jsonata/schema.ts +++ b/lib/modules/manager/custom/jsonata/schema.ts @@ -10,7 +10,7 @@ const DepObjectSchema = z.object({ depType: z.string().optional(), registryUrl: z.string().optional(), extractVersion: z.string().optional(), - indentation: z.number().optional(), + indentation: z.string().optional(), }); export const QueryResultZodSchema = z.array(DepObjectSchema); From 4e4739c6fb961edefd43493004509e4af641eb82 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 18 Dec 2024 23:32:32 +0530 Subject: [PATCH 27/37] validation for JSONata manager --- lib/config/options/index.ts | 10 ++++ lib/config/validation.spec.ts | 104 +++++++++++++++++++++++++++++++--- lib/config/validation.ts | 79 ++++++++++++++++++++++---- 3 files changed, 174 insertions(+), 19 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 6febe997bd5915..952c80f013e489 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2739,6 +2739,16 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'fileFormat', + description: + 'File format of the file which is targeted by the custom JSONata manager.', + type: 'string', + allowedValues: ['json'], + parents: ['customManagers'], + cli: false, + env: false, + }, { name: 'matchStrings', description: diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index a49e75c0de319e..29a3adac7d1db0 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -722,7 +722,8 @@ describe('config/validation', () => { currentValueTemplate: 'baz', }, { - customType: 'regex', + customType: 'jsonata', + fileFormat: 'json', fileMatch: ['foo'], depNameTemplate: 'foo', datasourceTemplate: 'bar', @@ -792,6 +793,60 @@ describe('config/validation', () => { expect(errors).toHaveLength(1); }); + it('error if no fileFormat in custom JSONata manager', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileMatch: ['package.json'], + matchStrings: [ + 'packages.{"depName": name, "currentValue": version, "datasource": "npm"}', + ], + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: 'Each JSONata manager must contain a fileFormat field.', + }, + ]); + }); + + it('validates JSONata query for each matchStrings', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: ['packages.{'], + depNameTemplate: 'foo', + datasourceTemplate: 'bar', + currentValueTemplate: 'baz', + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: `Invalid JSONata query for customManagers: \`packages.{\``, + }, + ]); + }); + // testing if we get all errors at once or not (possible), this does not include customType or fileMatch // since they are common to all custom managers it('validates all possible regex manager options', async () => { @@ -827,14 +882,12 @@ describe('config/validation', () => { depTypeTemplate: 'apple', }, { - customType: 'regex', - fileMatch: ['Dockerfile'], - matchStrings: ['ENV (?.*?)\\s'], - packageNameTemplate: 'foo', - datasourceTemplate: 'bar', - registryUrlTemplate: 'foobar', - extractVersionTemplate: '^(?v\\d+\\.\\d+)', - depTypeTemplate: 'apple', + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: [ + 'packages.{"depName": depName, "currentValue": version, "datasource": "npm"}', + ], }, ], }; @@ -892,6 +945,39 @@ describe('config/validation', () => { expect(errors).toHaveLength(1); }); + it('errors if customManager fields are missing: JSONataManager', async () => { + const config: RenovateConfig = { + customManagers: [ + { + customType: 'jsonata', + fileFormat: 'json', + fileMatch: ['package.json'], + matchStrings: ['packages'], + }, + ], + }; + const { warnings, errors } = await configValidation.validateConfig( + 'repo', + config, + true, + ); + expect(warnings).toHaveLength(0); + expect(errors).toMatchObject([ + { + topic: 'Configuration Error', + message: `JSONata Managers must contain currentValueTemplate configuration or currentValue in the query `, + }, + { + topic: 'Configuration Error', + message: `JSONata Managers must contain datasourceTemplate configuration or datasource in the query `, + }, + { + topic: 'Configuration Error', + message: `JSONata Managers must contain depName or packageName in the query or their templates`, + }, + ]); + }); + it('ignore keys', async () => { const config = { $schema: 'renovate.json', diff --git a/lib/config/validation.ts b/lib/config/validation.ts index d139cb42c10436..7a7b0704d892b7 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -1,11 +1,9 @@ import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; import { logger } from '../logger'; import { allManagersList, getManagerList } from '../modules/manager'; import { isCustomManager } from '../modules/manager/custom'; -import type { - RegexManagerConfig, - RegexManagerTemplates, -} from '../modules/manager/custom/regex/types'; +import type { RegexManagerTemplates } from '../modules/manager/custom/regex/types'; import type { CustomManager } from '../modules/manager/custom/types'; import type { HostRule } from '../types'; import { getExpression } from '../util/jsonata'; @@ -529,6 +527,7 @@ export async function validateConfig( const allowedKeys = [ 'customType', 'description', + 'fileFormat', 'fileMatch', 'matchStrings', 'matchStringsStrategy', @@ -570,6 +569,13 @@ export async function validateConfig( errors, ); break; + case 'jsonata': + validateJSONataManagerFields( + customManager, + currentPath, + errors, + ); + break; } } else { errors.push({ @@ -865,21 +871,21 @@ export async function validateConfig( return { errors, warnings }; } -function hasField( - customManager: Partial, - field: string, -): boolean { +function hasField(customManager: CustomManager, field: string): boolean { const templateField = `${field}Template` as keyof RegexManagerTemplates; + const fieldStr = + customManager.customType === 'regex' ? `(?<${field}>` : field; + return !!( customManager[templateField] ?? customManager.matchStrings?.some((matchString) => - matchString.includes(`(?<${field}>`), + matchString.includes(fieldStr), ) ); } function validateRegexManagerFields( - customManager: Partial, + customManager: CustomManager, currentPath: string, errors: ValidationMessage[], ): void { @@ -924,6 +930,59 @@ function validateRegexManagerFields( } } +function validateJSONataManagerFields( + customManager: CustomManager, + currentPath: string, + errors: ValidationMessage[], +): void { + if (!is.nonEmptyString(customManager.fileFormat)) { + errors.push({ + topic: 'Configuration Error', + message: 'Each JSONata manager must contain a fileFormat field.', + }); + } + + if (is.nonEmptyArray(customManager.matchStrings)) { + for (const matchString of customManager.matchStrings) { + try { + jsonata(matchString); + } catch (err) { + logger.debug( + { err }, + 'customManager.matchStrings JSONata query validation error', + ); + errors.push({ + topic: 'Configuration Error', + message: `Invalid JSONata query for ${currentPath}: \`${matchString}\``, + }); + } + } + } else { + errors.push({ + topic: 'Configuration Error', + message: `Each Custom Manager must contain a non-empty matchStrings array`, + }); + } + + const mandatoryFields = ['currentValue', 'datasource']; + for (const field of mandatoryFields) { + if (!hasField(customManager, field)) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain ${field}Template configuration or ${field} in the query `, + }); + } + } + + const nameFields = ['depName', 'packageName']; + if (!nameFields.some((field) => hasField(customManager, field))) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain depName or packageName in the query or their templates`, + }); + } +} + /** * Basic validation for global config options */ From e66b916326e4cc476fba43ba63eb876c2ea7c5b2 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Wed, 18 Dec 2024 23:33:03 +0530 Subject: [PATCH 28/37] docs(customManagers): fileFormat --- docs/usage/configuration-options.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 894f4bd3eca73c..0b1be6db29ddb9 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -805,6 +805,29 @@ Example: } ``` +### fileFormat + +File format of the file being targeted by the custom JSONata manager. +For example: if you are targeting a file like `.renovaterc` which allows both `yaml` and `json` syntax. +The bot needs the `fileFormat` specified to ease the parsing of that file. + +Currently, only the `json` format is supported. We intend to support more format in the future. + +```json +{ + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "fileMatch": ["file.json"], + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### datasourceTemplate If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. From e28ba8186cebb18d857e625d3bdd2cc62f20963c Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 00:29:52 +0530 Subject: [PATCH 29/37] Apply Suggestion --- .../manager/custom/jsonata/index.spec.ts | 37 +++++++------------ lib/modules/manager/custom/jsonata/index.ts | 8 +--- lib/modules/manager/custom/jsonata/utils.ts | 6 +-- lib/modules/manager/custom/utils.ts | 6 +++ 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 2f97272ae2776f..5818cd87c917d7 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -178,20 +178,18 @@ describe('modules/manager/custom/jsonata/index', () => { expect(res).toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), - 'Error while parsing dep info', + 'Query results failed schema validation', ); }); - it('returns null when content is not json', async () => { - const res = await extractPackageFile( - 'not-json', - 'foo-file', - {} as JsonataExtractConfig, - ); + it('returns null when content does not match specified file format', async () => { + const res = await extractPackageFile('not-json', 'foo-file', { + fileFormat: 'json', + } as JsonataExtractConfig); expect(res).toBeNull(); expect(logger.logger.warn).toHaveBeenCalledWith( expect.anything(), - 'Invalid JSON file(parsing failed)', + 'Error while parsing file', ); }); @@ -219,7 +217,9 @@ describe('modules/manager/custom/jsonata/index', () => { it('returns null if invalid template', async () => { const config = { fileFormat: 'json', - matchStrings: [`{"depName": "foo"}`], + matchStrings: [ + `{"depName": "foo", "currentValue": "1.0.0", "datasource": "npm"}`, + ], versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver', // invalid template }; const res = await extractPackageFile('{}', 'unused', config); @@ -233,7 +233,9 @@ describe('modules/manager/custom/jsonata/index', () => { it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => { const config = { fileFormat: 'json', - matchStrings: [`{"depName": "foo"}`], + matchStrings: [ + `{"depName": "foo", "currentValue": "1.0.0", "datasource": "npm"}`, + ], registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}', }; const res = await extractPackageFile('{}', 'unused', config); @@ -248,21 +250,10 @@ describe('modules/manager/custom/jsonata/index', () => { const config = { fileFormat: 'json', matchStrings: [`{"depName": "foo"}`, `{"depName": "bar"}`], + currentValueTemplate: '1.0.0', + datasourceTemplate: 'npm', }; const res = await extractPackageFile('{}', 'unused', config); expect(res?.deps).toHaveLength(2); }); - - it('excludes and warns if invalid jsonata query found', async () => { - const config = { - fileFormat: 'json', - matchStrings: ['{', `{"depName": "foo"}`, `{"depName": "bar"}`], - }; - const res = await extractPackageFile('{}', 'unused', config); - expect(res?.deps).toHaveLength(2); - expect(logger.logger.warn).toHaveBeenCalledWith( - { err: expect.any(Object), query: '{' }, - 'Failed to compile JSONata query', - ); - }); }); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 98ab5b67ab56d0..24fb671d08e089 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -25,15 +25,11 @@ export async function extractPackageFile( case 'json': json = parseJson(content, packageFile); break; - default: - throw new Error( - 'Invalid file format. JSONata only supports json files currently.', - ); } } catch (err) { logger.warn( - { err, fileName: packageFile }, - 'Invalid JSON file(parsing failed)', + { err, fileName: packageFile, fileFormat: config.fileFormat }, + 'Error while parsing file', ); return null; } diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index b31c65e52d1906..55b747ba138fbf 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -14,8 +14,8 @@ export async function handleMatching( packageFile: string, config: JsonataExtractConfig, ): Promise { - const results: Record[] = []; - const { matchStrings: jsonataQueries = [] } = config; + let results: Record[] = []; + const { matchStrings: jsonataQueries } = config; for (const query of jsonataQueries) { // won't fail as this is verified during config validation const jsonataExpression = jsonata(query); @@ -36,7 +36,7 @@ export async function handleMatching( queryResult = is.array(queryResult) ? queryResult : [queryResult]; const parsed = QueryResultZodSchema.safeParse(queryResult); if (parsed.success) { - results.concat(structuredClone(parsed.data)); + results = results.concat(structuredClone(parsed.data)); } else { logger.warn( { err: parsed.error, jsonataQuery: query, packageFile, queryResult }, diff --git a/lib/modules/manager/custom/utils.ts b/lib/modules/manager/custom/utils.ts index 024ac0d078d4b3..9d678173476abe 100644 --- a/lib/modules/manager/custom/utils.ts +++ b/lib/modules/manager/custom/utils.ts @@ -39,6 +39,8 @@ export function checkIsValidDependency( packageFile: string, manager: string, ): boolean { + // eslint-disable-next-line + console.log('checkIsValidDependency', dep); const isValid = isValidDependency(dep); if (!isValid) { const meta = { @@ -50,8 +52,12 @@ export function checkIsValidDependency( meta, 'Discovered a package dependency, but it did not pass validation. Discarding', ); + // eslint-disable-next-line + console.log('checkIsValidDependency:: 1'), isValid; return isValid; } + // eslint-disable-next-line + console.log('checkIsValidDependency:: 2', isValid); return isValid; } From 76b0cc4b04e47b6eda7ead688259262505d86d2f Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 00:38:44 +0530 Subject: [PATCH 30/37] fix issues --- docs/usage/configuration-options.md | 45 ++++++++++++++--------------- lib/config/options/index.ts | 2 +- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 0b1be6db29ddb9..a71d8048640354 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -805,29 +805,6 @@ Example: } ``` -### fileFormat - -File format of the file being targeted by the custom JSONata manager. -For example: if you are targeting a file like `.renovaterc` which allows both `yaml` and `json` syntax. -The bot needs the `fileFormat` specified to ease the parsing of that file. - -Currently, only the `json` format is supported. We intend to support more format in the future. - -```json -{ - "customManagers": [ - { - "customType": "jsonata", - "fileFormat": "json", - "fileMatch": ["file.json"], - "matchStrings": [ - "packages.{ \"depName\": package, \"currentValue\": version }" - ] - } - ] -} -``` - ### datasourceTemplate If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. @@ -848,6 +825,28 @@ It will be compiled using Handlebars and the regex `groups` result. If `extractVersion` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using Handlebars and the regex `groups` result. +### fileFormat + +It specifies the syntax of the package file being managed by the custom JSONata manager. +This setting helps the system correctly parse and interpret the configuration file's contents. + +Currently, only the `json` format is supported. We intend to support more formats in the future. + +```json +{ + "customManagers": [ + { + "customType": "jsonata", + "fileFormat": "json", + "fileMatch": [".renovaterc"], + "matchStrings": [ + "packages.{ \"depName\": package, \"currentValue\": version }" + ] + } + ] +} +``` + ### matchStrings Each `matchStrings` must be one of the two: diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 952c80f013e489..95e6f9768fe227 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2742,7 +2742,7 @@ const options: RenovateOptions[] = [ { name: 'fileFormat', description: - 'File format of the file which is targeted by the custom JSONata manager.', + 'It specifies the syntax of the package file being managed by the custom JSONata manager.', type: 'string', allowedValues: ['json'], parents: ['customManagers'], From 0e48b184a6a19379fb5d381b2b59180ad13bcf27 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 01:49:44 +0530 Subject: [PATCH 31/37] apply suggestions --- docs/usage/configuration-options.md | 4 ++-- lib/modules/manager/custom/utils.ts | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index a71d8048640354..1f3093e9944a23 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -706,7 +706,7 @@ You can define custom managers to handle: - Proprietary file formats or conventions - Popular file formats not yet supported as a manager by Renovate -Currently we only have two custom managers. +Currently we have two custom managers. The `regex` manager which is based on using Regular Expression named capture groups. The `jsonata` manager which is based on using JSONata queries. @@ -830,7 +830,7 @@ It will be compiled using Handlebars and the regex `groups` result. It specifies the syntax of the package file being managed by the custom JSONata manager. This setting helps the system correctly parse and interpret the configuration file's contents. -Currently, only the `json` format is supported. We intend to support more formats in the future. +Currently, only the `json` format is supported. ```json { diff --git a/lib/modules/manager/custom/utils.ts b/lib/modules/manager/custom/utils.ts index 9d678173476abe..024ac0d078d4b3 100644 --- a/lib/modules/manager/custom/utils.ts +++ b/lib/modules/manager/custom/utils.ts @@ -39,8 +39,6 @@ export function checkIsValidDependency( packageFile: string, manager: string, ): boolean { - // eslint-disable-next-line - console.log('checkIsValidDependency', dep); const isValid = isValidDependency(dep); if (!isValid) { const meta = { @@ -52,12 +50,8 @@ export function checkIsValidDependency( meta, 'Discovered a package dependency, but it did not pass validation. Discarding', ); - // eslint-disable-next-line - console.log('checkIsValidDependency:: 1'), isValid; return isValid; } - // eslint-disable-next-line - console.log('checkIsValidDependency:: 2', isValid); return isValid; } From f4ec39031875dc0639d6434437dcc7f0bd8f6785 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 17:32:37 +0530 Subject: [PATCH 32/37] Apply Suggestions --- docs/usage/configuration-options.md | 6 +-- lib/config/options/index.ts | 2 +- .../manager/custom/jsonata/index.spec.ts | 52 +++++++++++-------- lib/modules/manager/custom/jsonata/index.ts | 1 - lib/modules/manager/custom/jsonata/utils.ts | 6 +-- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 1f3093e9944a23..0642ae24f795cd 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -710,7 +710,7 @@ Currently we have two custom managers. The `regex` manager which is based on using Regular Expression named capture groups. The `jsonata` manager which is based on using JSONata queries. -You must have capture/extract the following three fields _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: +You must capture/extract the following three fields _or_ configure its corresponding template (e.g. `depNameTemplate`) for these fields: - `datasource` - `depName` and / or `packageName` @@ -790,7 +790,7 @@ Example: } ``` -```json +```json title="Parsing a JSON file with a custom manager" { "customManagers": [ { @@ -832,7 +832,7 @@ This setting helps the system correctly parse and interpret the configuration fi Currently, only the `json` format is supported. -```json +```json title="Parsing a JSON file with a custom manager" { "customManagers": [ { diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 95e6f9768fe227..6f6e919a4f93f1 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2752,7 +2752,7 @@ const options: RenovateOptions[] = [ { name: 'matchStrings', description: - 'Regex pattern or JSONata query to use. Valid only within a `customManagers` object.', + 'Queries to use. Valid only within a `customManagers` object. See `customType` docs.', type: 'array', subType: 'string', parents: ['customManagers'], diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index 5818cd87c917d7..bfa67088044dac 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -10,6 +10,27 @@ describe('modules/manager/custom/jsonata/index', () => { }); }); + it('returns null when content does not match specified file format', async () => { + const res = await extractPackageFile('not-json', 'foo-file', { + fileFormat: 'json', + } as JsonataExtractConfig); + expect(res).toBeNull(); + expect(logger.logger.warn).toHaveBeenCalledWith( + expect.anything(), + 'Error while parsing file', + ); + }); + + it('returns null when no content', async () => { + const res = await extractPackageFile('', 'foo-file', { + fileFormat: 'json', + matchStrings: [ + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + ], + } as JsonataExtractConfig); + expect(res).toBeNull(); + }); + it('extracts data when no templates are used', async () => { const json = codeBlock` { @@ -59,7 +80,6 @@ describe('modules/manager/custom/jsonata/index', () => { depType: 'dev', }, ], - matchStrings: config.matchStrings, }); }); @@ -142,7 +162,6 @@ describe('modules/manager/custom/jsonata/index', () => { depType: 'default-dep-type', }, ], - matchStrings: config.matchStrings, }); }); @@ -182,27 +201,6 @@ describe('modules/manager/custom/jsonata/index', () => { ); }); - it('returns null when content does not match specified file format', async () => { - const res = await extractPackageFile('not-json', 'foo-file', { - fileFormat: 'json', - } as JsonataExtractConfig); - expect(res).toBeNull(); - expect(logger.logger.warn).toHaveBeenCalledWith( - expect.anything(), - 'Error while parsing file', - ); - }); - - it('returns null when no content', async () => { - const res = await extractPackageFile('', 'foo-file', { - fileFormat: 'json', - matchStrings: [ - 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', - ], - } as JsonataExtractConfig); - expect(res).toBeNull(); - }); - it('returns null if no dependencies found', async () => { const config = { fileFormat: 'json', @@ -211,6 +209,14 @@ describe('modules/manager/custom/jsonata/index', () => { ], }; const res = await extractPackageFile('{}', 'unused', config); + expect(logger.logger.warn).toHaveBeenCalledWith( + { + packageFile: 'unused', + jsonataQuery: + 'packages.{ "depName": package, "currentValue": version, "versioning ": versioning }', + }, + 'The jsonata query returned no matches. Possible error, please check your query. Skipping', + ); expect(res).toBeNull(); }); diff --git a/lib/modules/manager/custom/jsonata/index.ts b/lib/modules/manager/custom/jsonata/index.ts index 24fb671d08e089..e40f6e19bb3f00 100644 --- a/lib/modules/manager/custom/jsonata/index.ts +++ b/lib/modules/manager/custom/jsonata/index.ts @@ -45,6 +45,5 @@ export async function extractPackageFile( return { deps, - matchStrings: config.matchStrings, }; } diff --git a/lib/modules/manager/custom/jsonata/utils.ts b/lib/modules/manager/custom/jsonata/utils.ts index 55b747ba138fbf..659c2bd97ec431 100644 --- a/lib/modules/manager/custom/jsonata/utils.ts +++ b/lib/modules/manager/custom/jsonata/utils.ts @@ -28,15 +28,15 @@ export async function handleMatching( jsonataQuery: query, packageFile, }, - 'The jsonata query returned no matches. Possible error, please check your query', + 'The jsonata query returned no matches. Possible error, please check your query. Skipping', ); - continue; + return []; } queryResult = is.array(queryResult) ? queryResult : [queryResult]; const parsed = QueryResultZodSchema.safeParse(queryResult); if (parsed.success) { - results = results.concat(structuredClone(parsed.data)); + results = results.concat(parsed.data); } else { logger.warn( { err: parsed.error, jsonataQuery: query, packageFile, queryResult }, From 66f2577afc11357f634d4cb11002166f080e5665 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 17:49:24 +0530 Subject: [PATCH 33/37] fix test --- lib/modules/manager/custom/jsonata/index.spec.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/custom/jsonata/index.spec.ts b/lib/modules/manager/custom/jsonata/index.spec.ts index bfa67088044dac..ed5d8fe5473632 100644 --- a/lib/modules/manager/custom/jsonata/index.spec.ts +++ b/lib/modules/manager/custom/jsonata/index.spec.ts @@ -260,6 +260,19 @@ describe('modules/manager/custom/jsonata/index', () => { datasourceTemplate: 'npm', }; const res = await extractPackageFile('{}', 'unused', config); - expect(res?.deps).toHaveLength(2); + expect(res).toMatchObject({ + deps: [ + { + depName: 'foo', + currentValue: '1.0.0', + datasource: 'npm', + }, + { + depName: 'bar', + currentValue: '1.0.0', + datasource: 'npm', + }, + ], + }); }); }); From d1da658df24f40f33afb1484b59bc16de4b197ac Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 20:31:49 +0530 Subject: [PATCH 34/37] rebase --- lib/config/validation-helpers/utils.ts | 57 ++++++++++++++++++++++++- lib/config/validation.ts | 59 +------------------------- 2 files changed, 57 insertions(+), 59 deletions(-) diff --git a/lib/config/validation-helpers/utils.ts b/lib/config/validation-helpers/utils.ts index 5d676bed324800..4912d34ce9f22b 100644 --- a/lib/config/validation-helpers/utils.ts +++ b/lib/config/validation-helpers/utils.ts @@ -1,9 +1,11 @@ import is from '@sindresorhus/is'; +import jsonata from 'jsonata'; import { logger } from '../../logger'; import type { RegexManagerConfig, RegexManagerTemplates, } from '../../modules/manager/custom/regex/types'; +import type { CustomManager } from '../../modules/manager/custom/types'; import { regEx } from '../../util/regex'; import type { ValidationMessage } from '../types'; @@ -92,7 +94,7 @@ function hasField( } export function validateRegexManagerFields( - customManager: Partial, + customManager: CustomManager, currentPath: string, errors: ValidationMessage[], ): void { @@ -136,3 +138,56 @@ export function validateRegexManagerFields( }); } } + +export function validateJSONataManagerFields( + customManager: CustomManager, + currentPath: string, + errors: ValidationMessage[], +): void { + if (!is.nonEmptyString(customManager.fileFormat)) { + errors.push({ + topic: 'Configuration Error', + message: 'Each JSONata manager must contain a fileFormat field.', + }); + } + + if (is.nonEmptyArray(customManager.matchStrings)) { + for (const matchString of customManager.matchStrings) { + try { + jsonata(matchString); + } catch (err) { + logger.debug( + { err }, + 'customManager.matchStrings JSONata query validation error', + ); + errors.push({ + topic: 'Configuration Error', + message: `Invalid JSONata query for ${currentPath}: \`${matchString}\``, + }); + } + } + } else { + errors.push({ + topic: 'Configuration Error', + message: `Each Custom Manager must contain a non-empty matchStrings array`, + }); + } + + const mandatoryFields = ['currentValue', 'datasource']; + for (const field of mandatoryFields) { + if (!hasField(customManager, field)) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain ${field}Template configuration or ${field} in the query `, + }); + } + } + + const nameFields = ['depName', 'packageName']; + if (!nameFields.some((field) => hasField(customManager, field))) { + errors.push({ + topic: 'Configuration Error', + message: `JSONata Managers must contain depName or packageName in the query or their templates`, + }); + } +} diff --git a/lib/config/validation.ts b/lib/config/validation.ts index cbc5017c2ee768..c89f07f4ee8e59 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -1,9 +1,4 @@ import is from '@sindresorhus/is'; -import jsonata from 'jsonata'; -import { logger } from '../logger'; -import { allManagersList, getManagerList } from '../modules/manager'; -import { isCustomManager } from '../modules/manager/custom'; -import type { RegexManagerTemplates } from '../modules/manager/custom/regex/types'; import { allManagersList, getManagerList } from '../modules/manager'; import { isCustomManager } from '../modules/manager/custom'; import type { CustomManager } from '../modules/manager/custom/types'; @@ -42,6 +37,7 @@ import * as regexOrGlobValidator from './validation-helpers/regex-glob-matchers' import { getParentName, isFalseGlobal, + validateJSONataManagerFields, validateNumber, validatePlainObject, validateRegexManagerFields, @@ -835,59 +831,6 @@ export async function validateConfig( return { errors, warnings }; } -function validateJSONataManagerFields( - customManager: CustomManager, - currentPath: string, - errors: ValidationMessage[], -): void { - if (!is.nonEmptyString(customManager.fileFormat)) { - errors.push({ - topic: 'Configuration Error', - message: 'Each JSONata manager must contain a fileFormat field.', - }); - } - - if (is.nonEmptyArray(customManager.matchStrings)) { - for (const matchString of customManager.matchStrings) { - try { - jsonata(matchString); - } catch (err) { - logger.debug( - { err }, - 'customManager.matchStrings JSONata query validation error', - ); - errors.push({ - topic: 'Configuration Error', - message: `Invalid JSONata query for ${currentPath}: \`${matchString}\``, - }); - } - } - } else { - errors.push({ - topic: 'Configuration Error', - message: `Each Custom Manager must contain a non-empty matchStrings array`, - }); - } - - const mandatoryFields = ['currentValue', 'datasource']; - for (const field of mandatoryFields) { - if (!hasField(customManager, field)) { - errors.push({ - topic: 'Configuration Error', - message: `JSONata Managers must contain ${field}Template configuration or ${field} in the query `, - }); - } - } - - const nameFields = ['depName', 'packageName']; - if (!nameFields.some((field) => hasField(customManager, field))) { - errors.push({ - topic: 'Configuration Error', - message: `JSONata Managers must contain depName or packageName in the query or their templates`, - }); - } -} - /** * Basic validation for global config options */ From e9ab58a85e024e2a068638085af4391d0a258a97 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 20:55:39 +0530 Subject: [PATCH 35/37] matchStrings: update description --- lib/config/options/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 6f6e919a4f93f1..92b3e29262054f 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2751,8 +2751,7 @@ const options: RenovateOptions[] = [ }, { name: 'matchStrings', - description: - 'Queries to use. Valid only within a `customManagers` object. See `customType` docs.', + description: 'Queries to use. Valid only within a `customManagers` object.', type: 'array', subType: 'string', parents: ['customManagers'], From a36550eb497228a84fe69905b45cbf80b831fae1 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Thu, 19 Dec 2024 20:59:19 +0530 Subject: [PATCH 36/37] update docs --- docs/usage/configuration-options.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 0642ae24f795cd..05c7d1c11a84fa 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -854,6 +854,8 @@ Each `matchStrings` must be one of the two: 1. a valid regular expression, optionally with named capture groups (if using `customType=regex`) 2. a valid, escaped [JSONata](https://docs.jsonata.org/overview.html) query (if using `customType=json`) +See [`customType`](#customtype) docs, to know more them. + Example: ```json title="matchStrings with a valid regular expression" From 37886613b855e172d171a8cc2db55f3a14922106 Mon Sep 17 00:00:00 2001 From: Rahul Gautam Singh Date: Sat, 21 Dec 2024 10:12:27 +0530 Subject: [PATCH 37/37] fix test --- lib/config/validation-helpers/utils.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/config/validation-helpers/utils.ts b/lib/config/validation-helpers/utils.ts index 4912d34ce9f22b..f10aec91ebfaf9 100644 --- a/lib/config/validation-helpers/utils.ts +++ b/lib/config/validation-helpers/utils.ts @@ -1,10 +1,7 @@ import is from '@sindresorhus/is'; import jsonata from 'jsonata'; import { logger } from '../../logger'; -import type { - RegexManagerConfig, - RegexManagerTemplates, -} from '../../modules/manager/custom/regex/types'; +import type { RegexManagerTemplates } from '../../modules/manager/custom/regex/types'; import type { CustomManager } from '../../modules/manager/custom/types'; import { regEx } from '../../util/regex'; import type { ValidationMessage } from '../types'; @@ -80,15 +77,14 @@ export function isFalseGlobal( return false; } -function hasField( - customManager: Partial, - field: string, -): boolean { +function hasField(customManager: CustomManager, field: string): boolean { const templateField = `${field}Template` as keyof RegexManagerTemplates; + const fieldStr = + customManager.customType === 'regex' ? `(?<${field}>` : field; return !!( customManager[templateField] ?? customManager.matchStrings?.some((matchString) => - matchString.includes(`(?<${field}>`), + matchString.includes(fieldStr), ) ); }