diff --git a/docs/06_custom_tags.md b/docs/06_custom_tags.md index 1364bb2b..afa6fb58 100644 --- a/docs/06_custom_tags.md +++ b/docs/06_custom_tags.md @@ -60,15 +60,16 @@ If including more than one custom tag from this set, make sure that the `'float' These tags are a part of the YAML 1.1 [language-independent types](https://yaml.org/type/), but are not a part of any default YAML 1.2 schema. -| Identifier | YAML Type | JS Type | Description | -| ------------- | ----------------------------------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `'binary'` | [`!!binary`](https://yaml.org/type/binary.html) | `Uint8Array` | Binary data, represented in YAML as base64 encoded characters. | -| `'floatTime'` | [`!!float`](https://yaml.org/type/float.html) | `Number` | Sexagesimal floating-point number format, e.g. `190:20:30.15`. To stringify with this tag, the node `format` must be `'TIME'`. | -| `'intTime'` | [`!!int`](https://yaml.org/type/int.html) | `Number` | Sexagesimal integer number format, e.g. `190:20:30`. To stringify with this tag, the node `format` must be `'TIME'`. | -| `'omap'` | [`!!omap`](https://yaml.org/type/omap.html) | `Map` | Ordered sequence of key: value pairs without duplicates. Using `mapAsMap: true` together with this tag is not recommended, as it makes the parse → stringify loop non-idempotent. | -| `'pairs'` | [`!!pairs`](https://yaml.org/type/pairs.html) | `Array` | Ordered sequence of key: value pairs allowing duplicates. To create from JS, use `doc.createNode(array, { tag: '!!pairs' })`. | -| `'set'` | [`!!set`](https://yaml.org/type/set.html) | `Set` | Unordered set of non-equal values. | -| `'timestamp'` | [`!!timestamp`](https://yaml.org/type/timestamp.html) | `Date` | A point in time, e.g. `2001-12-15T02:59:43`. | +| Identifier | YAML Type | JS Type | Description | +| ------------- | ----------------------------------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `'binary'` | [`!!binary`](https://yaml.org/type/binary.html) | `Uint8Array` | Binary data, represented in YAML as base64 encoded characters. | +| `'floatTime'` | [`!!float`](https://yaml.org/type/float.html) | `Number` | Sexagesimal floating-point number format, e.g. `190:20:30.15`. To stringify with this tag, the node `format` must be `'TIME'`. | +| `'intTime'` | [`!!int`](https://yaml.org/type/int.html) | `Number` | Sexagesimal integer number format, e.g. `190:20:30`. To stringify with this tag, the node `format` must be `'TIME'`. | +| `'merge'` | [`!!merge`](https://yaml.org/type/merge.html) | `Symbol('<<')` | A `<<` merge key which allows one or more mappings to be merged with the current one. | +| `'omap'` | [`!!omap`](https://yaml.org/type/omap.html) | `Map` | Ordered sequence of key: value pairs without duplicates. Using `mapAsMap: true` together with this tag is not recommended, as it makes the parse → stringify loop non-idempotent. | +| `'pairs'` | [`!!pairs`](https://yaml.org/type/pairs.html) | `Array` | Ordered sequence of key: value pairs allowing duplicates. To create from JS, use `doc.createNode(array, { tag: '!!pairs' })`. | +| `'set'` | [`!!set`](https://yaml.org/type/set.html) | `Set` | Unordered set of non-equal values. | +| `'timestamp'` | [`!!timestamp`](https://yaml.org/type/timestamp.html) | `Date` | A point in time, e.g. `2001-12-15T02:59:43`. | ## Writing Custom Tags diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts index 70a30ba8..0f2461c0 100644 --- a/src/compose/compose-doc.ts +++ b/src/compose/compose-doc.ts @@ -28,6 +28,7 @@ export function composeDoc< const opts = Object.assign({ _directives: directives }, options) const doc = new Document(undefined, opts) as Document.Parsed const ctx: ComposeContext = { + atKey: false, atRoot: true, directives: doc.directives, options: doc.options, diff --git a/src/compose/compose-node.ts b/src/compose/compose-node.ts index 2608d131..d07eb6d8 100644 --- a/src/compose/compose-node.ts +++ b/src/compose/compose-node.ts @@ -11,6 +11,7 @@ import { resolveEnd } from './resolve-end.js' import { emptyScalarPosition } from './util-empty-scalar-position.js' export interface ComposeContext { + atKey: boolean atRoot: boolean directives: Directives options: Readonly>> diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts index 86369f2b..71fbc896 100644 --- a/src/compose/compose-scalar.ts +++ b/src/compose/compose-scalar.ts @@ -87,14 +87,16 @@ function findScalarTagByName( } function findScalarTagByTest( - { directives, schema }: ComposeContext, + { atKey, directives, schema }: ComposeContext, value: string, token: FlowScalar, onError: ComposeErrorHandler ) { const tag = (schema.tags.find( - tag => tag.default && tag.test?.test(value) + tag => + (tag.default === true || (atKey && tag.default === 'key')) && + tag.test?.test(value) ) as ScalarTag) || schema[SCALAR] if (schema.compat) { diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts index 602bd38d..7e5abb00 100644 --- a/src/compose/resolve-block-map.ts +++ b/src/compose/resolve-block-map.ts @@ -69,11 +69,13 @@ export function resolveBlockMap( } // key value + ctx.atKey = true const keyStart = keyProps.end const keyNode = key ? composeNode(ctx, key, keyProps, onError) : composeEmptyNode(ctx, keyStart, start, null, keyProps, onError) if (ctx.schema.compat) flowIndentCheck(bm.indent, key, onError) + ctx.atKey = false if (mapIncludes(ctx, map.items, keyNode)) onError(keyStart, 'DUPLICATE_KEY', 'Map keys must be unique') diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts index 43e3a575..84aa3852 100644 --- a/src/compose/resolve-block-seq.ts +++ b/src/compose/resolve-block-seq.ts @@ -17,6 +17,7 @@ export function resolveBlockSeq( const seq = new NodeClass(ctx.schema) as YAMLSeq if (ctx.atRoot) ctx.atRoot = false + if (ctx.atKey) ctx.atKey = false let offset = bs.offset let commentEnd: number | null = null for (const { start, value } of bs.items) { diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts index 7e59af7a..9bab53f2 100644 --- a/src/compose/resolve-flow-collection.ts +++ b/src/compose/resolve-flow-collection.ts @@ -32,6 +32,7 @@ export function resolveFlowCollection( coll.flow = true const atRoot = ctx.atRoot if (atRoot) ctx.atRoot = false + if (ctx.atKey) ctx.atKey = false let offset = fc.offset + fc.start.source.length for (let i = 0; i < fc.items.length; ++i) { @@ -118,11 +119,13 @@ export function resolveFlowCollection( // item is a key+value pair // key value + ctx.atKey = true const keyStart = props.end const keyNode = key ? composeNode(ctx, key, props, onError) : composeEmptyNode(ctx, keyStart, start, null, props, onError) if (isBlock(key)) onError(keyNode.range, 'BLOCK_IN_FLOW', blockMsg) + ctx.atKey = false // value properties const valueProps = resolveProps(sep ?? [], { diff --git a/src/compose/util-map-includes.ts b/src/compose/util-map-includes.ts index e1e36a0f..ce20c118 100644 --- a/src/compose/util-map-includes.ts +++ b/src/compose/util-map-includes.ts @@ -14,10 +14,6 @@ export function mapIncludes( typeof uniqueKeys === 'function' ? uniqueKeys : (a: ParsedNode, b: ParsedNode) => - a === b || - (isScalar(a) && - isScalar(b) && - a.value === b.value && - !(a.value === '<<' && ctx.schema.merge)) + a === b || (isScalar(a) && isScalar(b) && a.value === b.value) return items.some(pair => isEqual(pair.key, search)) } diff --git a/src/doc/Document.ts b/src/doc/Document.ts index 5f0b5a74..d54d25f5 100644 --- a/src/doc/Document.ts +++ b/src/doc/Document.ts @@ -383,13 +383,13 @@ export class Document< case '1.1': if (this.directives) this.directives.yaml.version = '1.1' else this.directives = new Directives({ version: '1.1' }) - opt = { merge: true, resolveKnownTags: false, schema: 'yaml-1.1' } + opt = { resolveKnownTags: false, schema: 'yaml-1.1' } break case '1.2': case 'next': if (this.directives) this.directives.yaml.version = version else this.directives = new Directives({ version }) - opt = { merge: false, resolveKnownTags: true, schema: 'core' } + opt = { resolveKnownTags: true, schema: 'core' } break case null: if (this.directives) delete this.directives diff --git a/src/nodes/Node.ts b/src/nodes/Node.ts index 9ddef5e5..49efb66a 100644 --- a/src/nodes/Node.ts +++ b/src/nodes/Node.ts @@ -7,7 +7,7 @@ import type { Alias } from './Alias.js' import { isDocument, NODE_TYPE } from './identity.js' import type { Scalar } from './Scalar.js' import { toJS, ToJSContext } from './toJS.js' -import type { YAMLMap } from './YAMLMap.js' +import type { MapLike, YAMLMap } from './YAMLMap.js' import type { YAMLSeq } from './YAMLSeq.js' export type Node = @@ -70,6 +70,16 @@ export abstract class NodeBase { /** A fully qualified tag, if required */ declare tag?: string + /** + * Customize the way that a key-value pair is resolved. + * Used for YAML 1.1 !!merge << handling. + */ + declare addToJSMap?: ( + ctx: ToJSContext | undefined, + map: MapLike, + value: unknown + ) => void + /** A plain JS representation of this node */ abstract toJSON(): any diff --git a/src/nodes/addPairToJSMap.ts b/src/nodes/addPairToJSMap.ts index f6e8c133..4ee9a395 100644 --- a/src/nodes/addPairToJSMap.ts +++ b/src/nodes/addPairToJSMap.ts @@ -1,25 +1,20 @@ import { warn } from '../log.js' +import { addMergeToJSMap, isMergeKey } from '../schema/yaml-1.1/merge.js' import { createStringifyContext } from '../stringify/stringify.js' -import { isAlias, isMap, isNode, isScalar, isSeq } from './identity.js' +import { isNode } from './identity.js' import type { Pair } from './Pair.js' -import { Scalar } from './Scalar.js' import { toJS, ToJSContext } from './toJS.js' import type { MapLike } from './YAMLMap.js' -const MERGE_KEY = '<<' - export function addPairToJSMap( ctx: ToJSContext | undefined, map: MapLike, { key, value }: Pair ) { - if (ctx?.doc.schema.merge && isMergeKey(key)) { - value = isAlias(value) ? value.resolve(ctx.doc) : value - if (isSeq(value)) for (const it of value.items) mergeToJSMap(ctx, map, it) - else if (Array.isArray(value)) - for (const it of value) mergeToJSMap(ctx, map, it) - else mergeToJSMap(ctx, map, value) - } else { + if (isNode(key) && key.addToJSMap) key.addToJSMap(ctx, map, value) + // TODO: Should drop this special case for bare << handling + else if (isMergeKey(ctx, key)) addMergeToJSMap(ctx, map, value) + else { const jsKey = toJS(key, '', ctx) if (map instanceof Map) { map.set(jsKey, toJS(value, jsKey, ctx)) @@ -41,45 +36,6 @@ export function addPairToJSMap( return map } -const isMergeKey = (key: unknown) => - key === MERGE_KEY || - (isScalar(key) && - key.value === MERGE_KEY && - (!key.type || key.type === Scalar.PLAIN)) - -// If the value associated with a merge key is a single mapping node, each of -// its key/value pairs is inserted into the current mapping, unless the key -// already exists in it. If the value associated with the merge key is a -// sequence, then this sequence is expected to contain mapping nodes and each -// of these nodes is merged in turn according to its order in the sequence. -// Keys in mapping nodes earlier in the sequence override keys specified in -// later mapping nodes. -- http://yaml.org/type/merge.html -function mergeToJSMap( - ctx: ToJSContext | undefined, - map: MapLike, - value: unknown -) { - const source = ctx && isAlias(value) ? value.resolve(ctx.doc) : value - if (!isMap(source)) - throw new Error('Merge sources must be maps or map aliases') - const srcMap = source.toJSON(null, ctx, Map) - for (const [key, value] of srcMap) { - if (map instanceof Map) { - if (!map.has(key)) map.set(key, value) - } else if (map instanceof Set) { - map.add(key) - } else if (!Object.prototype.hasOwnProperty.call(map, key)) { - Object.defineProperty(map, key, { - value, - writable: true, - enumerable: true, - configurable: true - }) - } - } - return map -} - function stringifyKey( key: unknown, jsKey: unknown, diff --git a/src/schema/Schema.ts b/src/schema/Schema.ts index e16daa20..209e23bd 100644 --- a/src/schema/Schema.ts +++ b/src/schema/Schema.ts @@ -13,7 +13,6 @@ const sortMapEntriesByKey = (a: Pair, b: Pair) => export class Schema { compat: Array | null knownTags: Record - merge: boolean name: string sortMapEntries: ((a: Pair, b: Pair) => number) | null tags: Array @@ -38,10 +37,9 @@ export class Schema { : compat ? getTags(null, compat) : null - this.merge = !!merge this.name = (typeof schema === 'string' && schema) || 'core' this.knownTags = resolveKnownTags ? coreKnownTags : {} - this.tags = getTags(customTags, this.name) + this.tags = getTags(customTags, this.name, merge) this.toStringOptions = toStringDefaults ?? null Object.defineProperty(this, MAP, { value: map }) diff --git a/src/schema/tags.ts b/src/schema/tags.ts index 8ddfed38..483f9169 100644 --- a/src/schema/tags.ts +++ b/src/schema/tags.ts @@ -9,6 +9,7 @@ import { int, intHex, intOct } from './core/int.js' import { schema as core } from './core/schema.js' import { schema as json } from './json/schema.js' import { binary } from './yaml-1.1/binary.js' +import { merge } from './yaml-1.1/merge.js' import { omap } from './yaml-1.1/omap.js' import { pairs } from './yaml-1.1/pairs.js' import { schema as yaml11 } from './yaml-1.1/schema.js' @@ -36,6 +37,7 @@ const tagsByName = { intOct, intTime, map, + merge, null: nullTag, omap, pairs, @@ -50,6 +52,7 @@ export type Tags = Array export const coreKnownTags = { 'tag:yaml.org,2002:binary': binary, + 'tag:yaml.org,2002:merge': merge, 'tag:yaml.org,2002:omap': omap, 'tag:yaml.org,2002:pairs': pairs, 'tag:yaml.org,2002:set': set, @@ -58,9 +61,17 @@ export const coreKnownTags = { export function getTags( customTags: SchemaOptions['customTags'] | undefined, - schemaName: string + schemaName: string, + addMergeTag?: boolean ) { - let tags: Tags | undefined = schemas.get(schemaName) + const schemaTags = schemas.get(schemaName) + if (schemaTags && !customTags) { + return addMergeTag && !schemaTags.includes(merge) + ? schemaTags.concat(merge) + : schemaTags.slice() + } + + let tags: Tags | undefined = schemaTags if (!tags) { if (Array.isArray(customTags)) tags = [] else { @@ -79,14 +90,18 @@ export function getTags( } else if (typeof customTags === 'function') { tags = customTags(tags.slice()) } + if (addMergeTag) tags = tags.concat(merge) - return tags.map(tag => { - if (typeof tag !== 'string') return tag - const tagObj = tagsByName[tag] - if (tagObj) return tagObj - const keys = Object.keys(tagsByName) - .map(key => JSON.stringify(key)) - .join(', ') - throw new Error(`Unknown custom tag "${tag}"; use one of ${keys}`) - }) + return tags.reduce<(CollectionTag | ScalarTag)[]>((tags, tag) => { + const tagObj = typeof tag === 'string' ? tagsByName[tag] : tag + if (!tagObj) { + const tagName = JSON.stringify(tag) + const keys = Object.keys(tagsByName) + .map(key => JSON.stringify(key)) + .join(', ') + throw new Error(`Unknown custom tag ${tagName}; use one of ${keys}`) + } + if (!tags.includes(tagObj)) tags.push(tagObj) + return tags + }, []) } diff --git a/src/schema/types.ts b/src/schema/types.ts index 1bccc0bc..0a400176 100644 --- a/src/schema/types.ts +++ b/src/schema/types.ts @@ -14,11 +14,13 @@ interface TagBase { createNode?: (schema: Schema, value: unknown, ctx: CreateNodeContext) => Node /** - * If `true`, together with `test` allows for values to be stringified without - * an explicit tag. For most cases, it's unlikely that you'll actually want to - * use this, even if you first think you do. + * If `true`, allows for values to be stringified without + * an explicit tag together with `test`. + * If `'key'`, this only applies if the value is used as a mapping key. + * For most cases, it's unlikely that you'll actually want to use this, + * even if you first think you do. */ - default?: boolean + default?: boolean | 'key' /** * If a tag has multiple forms that should be parsed and/or stringified diff --git a/src/schema/yaml-1.1/merge.ts b/src/schema/yaml-1.1/merge.ts new file mode 100644 index 00000000..68d4bce5 --- /dev/null +++ b/src/schema/yaml-1.1/merge.ts @@ -0,0 +1,77 @@ +import { isAlias, isMap, isScalar, isSeq } from '../../nodes/identity.js' +import { Scalar } from '../../nodes/Scalar.js' +import type { ToJSContext } from '../../nodes/toJS.js' +import type { MapLike } from '../../nodes/YAMLMap.js' +import type { ScalarTag } from '../types.js' + +// If the value associated with a merge key is a single mapping node, each of +// its key/value pairs is inserted into the current mapping, unless the key +// already exists in it. If the value associated with the merge key is a +// sequence, then this sequence is expected to contain mapping nodes and each +// of these nodes is merged in turn according to its order in the sequence. +// Keys in mapping nodes earlier in the sequence override keys specified in +// later mapping nodes. -- http://yaml.org/type/merge.html + +const MERGE_KEY = '<<' + +export const merge: ScalarTag & { + identify(value: unknown): boolean + test: RegExp +} = { + identify: value => + value === MERGE_KEY || + (typeof value === 'symbol' && value.description === MERGE_KEY), + default: 'key', + tag: 'tag:yaml.org,2002:merge', + test: /^<<$/, + resolve: () => + Object.assign(new Scalar(Symbol(MERGE_KEY)), { + addToJSMap: addMergeToJSMap + }), + stringify: () => MERGE_KEY +} + +export const isMergeKey = (ctx: ToJSContext | undefined, key: unknown) => + (merge.identify(key) || + (isScalar(key) && + (!key.type || key.type === Scalar.PLAIN) && + merge.identify(key.value))) && + ctx?.doc.schema.tags.some(tag => tag.tag === merge.tag && tag.default) + +export function addMergeToJSMap( + ctx: ToJSContext | undefined, + map: MapLike, + value: unknown +) { + value = ctx && isAlias(value) ? value.resolve(ctx.doc) : value + if (isSeq(value)) for (const it of value.items) mergeValue(ctx, map, it) + else if (Array.isArray(value)) + for (const it of value) mergeValue(ctx, map, it) + else mergeValue(ctx, map, value) +} + +function mergeValue( + ctx: ToJSContext | undefined, + map: MapLike, + value: unknown +) { + const source = ctx && isAlias(value) ? value.resolve(ctx.doc) : value + if (!isMap(source)) + throw new Error('Merge sources must be maps or map aliases') + const srcMap = source.toJSON(null, ctx, Map) + for (const [key, value] of srcMap) { + if (map instanceof Map) { + if (!map.has(key)) map.set(key, value) + } else if (map instanceof Set) { + map.add(key) + } else if (!Object.prototype.hasOwnProperty.call(map, key)) { + Object.defineProperty(map, key, { + value, + writable: true, + enumerable: true, + configurable: true + }) + } + } + return map +} diff --git a/src/schema/yaml-1.1/schema.ts b/src/schema/yaml-1.1/schema.ts index 7a5f3160..c7b700b9 100644 --- a/src/schema/yaml-1.1/schema.ts +++ b/src/schema/yaml-1.1/schema.ts @@ -6,6 +6,7 @@ import { binary } from './binary.js' import { falseTag, trueTag } from './bool.js' import { float, floatExp, floatNaN } from './float.js' import { intBin, int, intHex, intOct } from './int.js' +import { merge } from './merge.js' import { omap } from './omap.js' import { pairs } from './pairs.js' import { set } from './set.js' @@ -26,6 +27,7 @@ export const schema = [ floatExp, float, binary, + merge, omap, pairs, set, diff --git a/src/stringify/stringify.ts b/src/stringify/stringify.ts index c05e8858..0c454709 100644 --- a/src/stringify/stringify.ts +++ b/src/stringify/stringify.ts @@ -96,7 +96,11 @@ function getTagObject(tags: Array, item: Node) { let obj: unknown if (isScalar(item)) { obj = item.value - const match = tags.filter(t => t.identify?.(obj)) + let match = tags.filter(t => t.identify?.(obj)) + if (match.length > 1) { + const testMatch = match.filter(t => t.test) + if (testMatch.length > 0) match = testMatch + } tagObj = match.find(t => t.format === item.format) ?? match.find(t => !t.format) } else { diff --git a/tests/doc/anchors.ts b/tests/doc/anchors.ts index 9ea4cd06..70a44da7 100644 --- a/tests/doc/anchors.ts +++ b/tests/doc/anchors.ts @@ -211,12 +211,12 @@ describe('merge <<', () => { << : [ *CENTER, *BIG ] label: center/big -- # Override - << : [ *BIG, *LEFT, *SMALL ] +- # Override with explicit tag + !!merge << : [ *BIG, *LEFT, *SMALL ] x: 1 label: center/big` - test('YAML.parse', () => { + test('YAML.parse with merge:true', () => { const res = parse(src, { merge: true }) expect(res).toHaveLength(8) for (let i = 4; i < res.length; ++i) { @@ -224,12 +224,21 @@ describe('merge <<', () => { } }) + test('YAML.parse with customTags:["merge"]', () => { + const res = parse(src, { customTags: ['merge'] }) + expect(res).toHaveLength(8) + for (let i = 4; i < res.length; ++i) { + expect(res[i]).toMatchObject({ x: 1, y: 2, r: 10, label: 'center/big' }) + } + }) + test('YAML.parse with merge:false', () => { const res = parse(src) expect(res).toHaveLength(8) - for (let i = 5; i < res.length; ++i) { + for (let i = 5; i < res.length - 1; ++i) { expect(res[i]).toHaveProperty('<<') } + expect(res.at(-1)).toMatchObject({ x: 1, y: 2, r: 10, label: 'center/big' }) }) test('YAML.parseDocument', () => { @@ -284,6 +293,30 @@ describe('merge <<', () => { expect(String(doc)).toBe('[ &a1 { a: A }, { b: B, <<: *a1 } ]\n') }) + test('using customTags:["merge"]', () => { + const doc = parseDocument, false>( + '[{ a: A }, { b: B }]', + { customTags: ['merge'] } + ) + const [a, b] = doc.contents.items + const merge = doc.createPair('<<', doc.createAlias(a)) + b.items.push(merge) + expect(doc.toJS()).toMatchObject([{ a: 'A' }, { a: 'A', b: 'B' }]) + expect(String(doc)).toBe('[ &a1 { a: A }, { b: B, <<: *a1 } ]\n') + }) + + test('symbol value', () => { + const doc = parseDocument, false>( + '[{ a: A }, { b: B }]', + { merge: true } + ) + const [a, b] = doc.contents.items + const merge = doc.createPair(Symbol('<<'), doc.createAlias(a)) + b.items.push(merge) + expect(doc.toJS()).toMatchObject([{ a: 'A' }, { a: 'A', b: 'B' }]) + expect(String(doc)).toBe('[ &a1 { a: A }, { b: B, <<: *a1 } ]\n') + }) + test('merge pair of an alias', () => { const doc = parseDocument, false>( '[{ a: A }, { b: B }]',