From 491945bd8a03f215add2a51b5bd9aa4f7ab12adc Mon Sep 17 00:00:00 2001 From: Florent Bouisset <58945185+Florent-Bouisset@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:49:32 +0100 Subject: [PATCH] Use SCTE-214 supplemental codec instead of base codec if it's supported * parse supplemental codec * add a property isSupplementalCoded supported and update getMimeType * add documentation for isSupplementalCodecSupported property * parse supplementalCodec in the adaptation set * return the codec that will be played if both supplemental codec is supported * parse multiple supplemental codec separated by white space * avoid unecessary call to iseCodecSupported if supplementalCodec is true * remove test file that shouldn't have been add * remplace == with isNullOrUndefined function * update regex to match more unexpected whitespace * code review * update rust WASM parser to parse supplementalCodecs * doc: update rfc number that obsoletes rfc4281 * move codec converting logic in "common" to convert from any parser * code review * set the codec to supplementalCodec if supported and remove supplementalCodec in the representation * handle case for subtitle * move code to another folder * reset unchanged file * remove staged file --- src/manifest/representation.ts | 25 +++++++--- .../convert_supplemental_codecs.test.ts | 37 +++++++++++++++ .../common/convert_supplemental_codecs.ts | 32 +++++++++++++ .../dash/common/parse_representations.ts | 47 ++++++++++++------- .../js-parser/node_parsers/AdaptationSet.ts | 4 ++ .../js-parser/node_parsers/Representation.ts | 4 ++ .../manifest/dash/node_parser_types.ts | 2 + .../manifest/dash/wasm-parser/rs/events.rs | 2 + .../wasm-parser/rs/processor/attributes.rs | 2 + .../ts/generators/AdaptationSet.ts | 4 ++ .../ts/generators/Representation.ts | 4 ++ .../manifest/dash/wasm-parser/ts/types.ts | 2 + src/parsers/manifest/types.ts | 2 + 13 files changed, 144 insertions(+), 23 deletions(-) create mode 100644 src/parsers/manifest/dash/common/__tests__/convert_supplemental_codecs.test.ts create mode 100644 src/parsers/manifest/dash/common/convert_supplemental_codecs.ts diff --git a/src/manifest/representation.ts b/src/manifest/representation.ts index 5c2aae8f2f..7a6c103352 100644 --- a/src/manifest/representation.ts +++ b/src/manifest/representation.ts @@ -190,15 +190,28 @@ class Representation { } this.cdnMetadata = args.cdnMetadata; - this.index = args.index; + if (opts.type === "audio" || opts.type === "video") { - const mimeTypeStr = this.getMimeTypeString(); - const isSupported = isCodecSupported(mimeTypeStr); - if (!isSupported) { - log.info("Unsupported Representation", mimeTypeStr, this.id, this.bitrate); + this.isSupported = false; + // Supplemental codecs are defined as backwards-compatible codecs enhancing + // the experience of a base layer codec + if (args.supplementalCodecs !== undefined) { + const supplementalCodecMimeTypeStr = + `${this.mimeType ?? ""};codecs="${args.supplementalCodecs}"`; + if (isCodecSupported(supplementalCodecMimeTypeStr)) { + this.codec = args.supplementalCodecs; + this.isSupported = true; + } + } + if (!this.isSupported) { + const mimeTypeStr = this.getMimeTypeString(); + const isSupported = isCodecSupported(mimeTypeStr); + if (!isSupported) { + log.info("Unsupported Representation", mimeTypeStr, this.id, this.bitrate); + } + this.isSupported = isSupported; } - this.isSupported = isSupported; } else { this.isSupported = true; // TODO for other types } diff --git a/src/parsers/manifest/dash/common/__tests__/convert_supplemental_codecs.test.ts b/src/parsers/manifest/dash/common/__tests__/convert_supplemental_codecs.test.ts new file mode 100644 index 0000000000..7bf5e96a8b --- /dev/null +++ b/src/parsers/manifest/dash/common/__tests__/convert_supplemental_codecs.test.ts @@ -0,0 +1,37 @@ +import { convertSupplementalCodecsToRFC6381 } from "../convert_supplemental_codecs"; + +describe("parseSupplementalCodec", () => { + it("should return the codec unchanged if there is only one codec", () => { + expect(convertSupplementalCodecsToRFC6381("avc1.4d400d")) + .toEqual("avc1.4d400d"); + }); + it("should trim starting and ending whitespace", () => { + expect( + convertSupplementalCodecsToRFC6381(" avc1.4d400d ")) + .toEqual("avc1.4d400d"); + }); + it("should return comma-separated list if input is whitespace-separated", () => { + expect( + convertSupplementalCodecsToRFC6381("avc1.4d400d avc1.4d4015")) + .toEqual("avc1.4d400d, avc1.4d4015"); + }); + it("should return comma-separated value if input is already comma-separated", () => { + expect( + convertSupplementalCodecsToRFC6381("avc1.4d400d, avc1.4d4015")) + .toEqual("avc1.4d400d, avc1.4d4015"); + }); + + it("should return comma-separated value if input as missplaced whitespace", () => { + expect( + convertSupplementalCodecsToRFC6381("avc1.4d400d , avc1.4d4015 ")) + .toEqual("avc1.4d400d, avc1.4d4015"); + }); + + it(`should return comma-separated value if input is mix of comma and + whitespace separated list` + , () => { + expect( + convertSupplementalCodecsToRFC6381("avc1.4d400d avc1.4d4015, avc1.4d401f")) + .toEqual("avc1.4d400d, avc1.4d4015, avc1.4d401f"); + }); +}); diff --git a/src/parsers/manifest/dash/common/convert_supplemental_codecs.ts b/src/parsers/manifest/dash/common/convert_supplemental_codecs.ts new file mode 100644 index 0000000000..ae201ac694 --- /dev/null +++ b/src/parsers/manifest/dash/common/convert_supplemental_codecs.ts @@ -0,0 +1,32 @@ +import isNonEmptyString from "../../../../utils/is_non_empty_string"; + +const supplementalCodecSeparator = /[, ]+/g; +/** + * Converts SCTE 214 supplemental codec string into RFC4281 codec string + * + * The returned value is a codec string respecting RFC6381 + * + * SCTE 214 defines supplemental codecs as a whitespace-separated multiple list of + * codec strings + * + * RFC6381 defines codecs as a comma-separated list of codec strings. + * + * This two syntax differs and this parser is used to convert SCTE214 + * to be compliant with what MSE APIs expect + * + * @param {string} val - The codec string to parse + * @returns { Array.} + */ +export function convertSupplementalCodecsToRFC6381( + val: string +) : string { + + if (isNonEmptyString(val)) { + return val + .trim() + .replace(supplementalCodecSeparator, ", "); + } + return ""; +} + + diff --git a/src/parsers/manifest/dash/common/parse_representations.ts b/src/parsers/manifest/dash/common/parse_representations.ts index 8379a7a3f2..0323d6d0e6 100644 --- a/src/parsers/manifest/dash/common/parse_representations.ts +++ b/src/parsers/manifest/dash/common/parse_representations.ts @@ -29,6 +29,7 @@ import { IScheme, IContentProtectionIntermediateRepresentation, } from "../node_parser_types"; +import { convertSupplementalCodecsToRFC6381 } from "./convert_supplemental_codecs"; import { getWEBMHDRInformation } from "./get_hdr_information"; import parseRepresentationIndex, { IRepresentationIndexContext, @@ -120,19 +121,19 @@ export default function parseRepresentations( const parsedRepresentations : IParsedRepresentation[] = []; for (const representation of representationsIR) { // Compute Representation ID - let representationID = representation.attributes.id != null ? + let representationID = representation.attributes.id !== undefined ? representation.attributes.id : (String(representation.attributes.bitrate) + - (representation.attributes.height != null ? + (representation.attributes.height !== undefined ? (`-${representation.attributes.height}`) : "") + - (representation.attributes.width != null ? + (representation.attributes.width !== undefined ? (`-${representation.attributes.width}`) : "") + - (representation.attributes.mimeType != null ? + (representation.attributes.mimeType !== undefined ? (`-${representation.attributes.mimeType}`) : "") + - (representation.attributes.codecs != null ? + (representation.attributes.codecs !== undefined ? (`-${representation.attributes.codecs}`) : "")); @@ -167,7 +168,7 @@ export default function parseRepresentations( // Find bitrate let representationBitrate : number; - if (representation.attributes.bitrate == null) { + if (representation.attributes.bitrate === undefined) { log.warn("DASH: No usable bitrate found in the Representation."); representationBitrate = 0; } else { @@ -204,40 +205,52 @@ export default function parseRepresentations( // Add optional attributes let codecs : string|undefined; - if (representation.attributes.codecs != null) { + if (representation.attributes.codecs !== undefined) { codecs = representation.attributes.codecs; - } else if (adaptation.attributes.codecs != null) { + } else if (adaptation.attributes.codecs !== undefined) { codecs = adaptation.attributes.codecs; } - if (codecs != null) { + if (codecs !== undefined) { codecs = codecs === "mp4a.40.02" ? "mp4a.40.2" : codecs; parsedRepresentation.codecs = codecs; } - if (representation.attributes.frameRate != null) { + + let supplementalCodecs: string | undefined; + if (representation.attributes.supplementalCodecs !== undefined) { + supplementalCodecs = representation.attributes.supplementalCodecs; + } else if (adaptation.attributes.supplementalCodecs !== undefined) { + supplementalCodecs = adaptation.attributes.supplementalCodecs; + } + if (supplementalCodecs !== undefined) { + parsedRepresentation.supplementalCodecs = + convertSupplementalCodecsToRFC6381(supplementalCodecs); + } + + if (representation.attributes.frameRate !== undefined) { parsedRepresentation.frameRate = representation.attributes.frameRate; - } else if (adaptation.attributes.frameRate != null) { + } else if (adaptation.attributes.frameRate !== undefined) { parsedRepresentation.frameRate = adaptation.attributes.frameRate; } - if (representation.attributes.height != null) { + if (representation.attributes.height !== undefined) { parsedRepresentation.height = representation.attributes.height; - } else if (adaptation.attributes.height != null) { + } else if (adaptation.attributes.height !== undefined) { parsedRepresentation.height = adaptation.attributes.height; } - if (representation.attributes.mimeType != null) { + if (representation.attributes.mimeType !== undefined) { parsedRepresentation.mimeType = representation.attributes.mimeType; - } else if (adaptation.attributes.mimeType != null) { + } else if (adaptation.attributes.mimeType !== undefined) { parsedRepresentation.mimeType = adaptation.attributes.mimeType; } - if (representation.attributes.width != null) { + if (representation.attributes.width !== undefined) { parsedRepresentation.width = representation.attributes.width; - } else if (adaptation.attributes.width != null) { + } else if (adaptation.attributes.width !== undefined) { parsedRepresentation.width = adaptation.attributes.width; } diff --git a/src/parsers/manifest/dash/js-parser/node_parsers/AdaptationSet.ts b/src/parsers/manifest/dash/js-parser/node_parsers/AdaptationSet.ts index 1ced658909..004c42c705 100644 --- a/src/parsers/manifest/dash/js-parser/node_parsers/AdaptationSet.ts +++ b/src/parsers/manifest/dash/js-parser/node_parsers/AdaptationSet.ts @@ -296,6 +296,10 @@ function parseAdaptationSetAttributes( parsedAdaptation.codecs = attribute.value; break; + case "scte214:supplementalCodecs": + parsedAdaptation.supplementalCodecs = attribute.value; + break; + case "codingDependency": parseValue(attribute.value, { asKey: "codingDependency", parser: parseBoolean, diff --git a/src/parsers/manifest/dash/js-parser/node_parsers/Representation.ts b/src/parsers/manifest/dash/js-parser/node_parsers/Representation.ts index 2f74e98586..5975b51a01 100644 --- a/src/parsers/manifest/dash/js-parser/node_parsers/Representation.ts +++ b/src/parsers/manifest/dash/js-parser/node_parsers/Representation.ts @@ -184,6 +184,10 @@ function parseRepresentationAttributes( dashName: "qualityRanking" }); break; + case "scte214:supplementalCodecs": + attributes.supplementalCodecs = attr.value; + break; + case "segmentProfiles": attributes.segmentProfiles = attr.value; break; diff --git a/src/parsers/manifest/dash/node_parser_types.ts b/src/parsers/manifest/dash/node_parser_types.ts index dc24b076dd..6b66d4e739 100644 --- a/src/parsers/manifest/dash/node_parser_types.ts +++ b/src/parsers/manifest/dash/node_parser_types.ts @@ -232,6 +232,7 @@ export interface IAdaptationSetAttributes { segmentAlignment? : number|boolean; segmentProfiles? : string; subsegmentAlignment? : number|boolean; + supplementalCodecs?: string; width? : number; availabilityTimeComplete?: boolean; availabilityTimeOffset?: number; @@ -271,6 +272,7 @@ export interface IRepresentationAttributes { profiles? : string; qualityRanking? : number; segmentProfiles? : string; + supplementalCodecs?: string; width? : number; availabilityTimeComplete?: boolean; availabilityTimeOffset?: number; diff --git a/src/parsers/manifest/dash/wasm-parser/rs/events.rs b/src/parsers/manifest/dash/wasm-parser/rs/events.rs index 27b1a81a87..6a0505c7db 100644 --- a/src/parsers/manifest/dash/wasm-parser/rs/events.rs +++ b/src/parsers/manifest/dash/wasm-parser/rs/events.rs @@ -280,6 +280,8 @@ pub enum AttributeName { // SegmentTemplate EndNumber = 76, // f64 + + SupplementalCodecs = 77 // string } impl TagName { diff --git a/src/parsers/manifest/dash/wasm-parser/rs/processor/attributes.rs b/src/parsers/manifest/dash/wasm-parser/rs/processor/attributes.rs index b9a5cf4591..a8f9610222 100644 --- a/src/parsers/manifest/dash/wasm-parser/rs/processor/attributes.rs +++ b/src/parsers/manifest/dash/wasm-parser/rs/processor/attributes.rs @@ -87,6 +87,7 @@ pub fn report_adaptation_set_attrs(e: &quick_xml::events::BytesStart) { b"bitstreamSwitching" => BitstreamSwitching.try_report_as_bool(&attr), b"audioSamplingRate" => AudioSamplingRate.try_report_as_string(&attr), b"codecs" => Codecs.try_report_as_string(&attr), + b"scte214:supplementalCodecs" => SupplementalCodecs.try_report_as_string(&attr), b"profiles" => Profiles.try_report_as_string(&attr), b"segmentProfiles" => SegmentProfiles.try_report_as_string(&attr), b"mimeType" => MimeType.try_report_as_string(&attr), @@ -116,6 +117,7 @@ pub fn report_representation_attrs(tag_bs: &quick_xml::events::BytesStart) { b"audioSamplingRate" => AudioSamplingRate.try_report_as_string(&attr), b"bandwidth" => Bitrate.try_report_as_u64(&attr), b"codecs" => Codecs.try_report_as_string(&attr), + b"scte214:supplementalCodecs" => SupplementalCodecs.try_report_as_string(&attr), b"codingDependency" => CodingDependency.try_report_as_bool(&attr), b"frameRate" => FrameRate.try_report_as_string(&attr), b"height" => Height.try_report_as_u64(&attr), diff --git a/src/parsers/manifest/dash/wasm-parser/ts/generators/AdaptationSet.ts b/src/parsers/manifest/dash/wasm-parser/ts/generators/AdaptationSet.ts index 983a7b65bf..cb73099787 100644 --- a/src/parsers/manifest/dash/wasm-parser/ts/generators/AdaptationSet.ts +++ b/src/parsers/manifest/dash/wasm-parser/ts/generators/AdaptationSet.ts @@ -289,6 +289,10 @@ export function generateAdaptationSetAttrParser( adaptationAttrs.codecs = parseString(textDecoder, linearMemory.buffer, ptr, len); break; + case AttributeName.SupplementalCodecs: + adaptationAttrs.supplementalCodecs = + parseString(textDecoder, linearMemory.buffer, ptr, len); + break; case AttributeName.Profiles: adaptationAttrs.profiles = parseString(textDecoder, linearMemory.buffer, ptr, len); diff --git a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts index 266a16b630..4836cce0b1 100644 --- a/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts +++ b/src/parsers/manifest/dash/wasm-parser/ts/generators/Representation.ts @@ -165,6 +165,10 @@ export function generateRepresentationAttrParser( representationAttrs.codecs = parseString(textDecoder, linearMemory.buffer, ptr, len); break; + case AttributeName.SupplementalCodecs: + representationAttrs.supplementalCodecs = + parseString(textDecoder, linearMemory.buffer, ptr, len); + break; case AttributeName.CodingDependency: representationAttrs.codingDependency = new DataView(linearMemory.buffer).getUint8(0) === 0; diff --git a/src/parsers/manifest/dash/wasm-parser/ts/types.ts b/src/parsers/manifest/dash/wasm-parser/ts/types.ts index ef0733f47a..e9320c0678 100644 --- a/src/parsers/manifest/dash/wasm-parser/ts/types.ts +++ b/src/parsers/manifest/dash/wasm-parser/ts/types.ts @@ -293,4 +293,6 @@ export const enum AttributeName { // SegmentTemplate EndNumber = 76, // f64 + + SupplementalCodecs = 77, // String } diff --git a/src/parsers/manifest/types.ts b/src/parsers/manifest/types.ts index 28b8caa0f4..0d958ecaa8 100644 --- a/src/parsers/manifest/types.ts +++ b/src/parsers/manifest/types.ts @@ -160,6 +160,8 @@ export interface IParsedRepresentation { hdrInfo?: IHDRInformation | undefined; /** `true` if audio has Dolby Atmos. */ isSpatialAudio?: boolean | undefined; + + supplementalCodecs? : string | undefined; } /** Every possible types an Adaptation can have. */