Source: lib/cea/mp4_cea_parser.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.cea.Mp4CeaParser');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.cea.CeaUtils');
  11. goog.require('shaka.cea.SeiProcessor');
  12. goog.require('shaka.log');
  13. goog.require('shaka.media.ClosedCaptionParser');
  14. goog.require('shaka.util.DataViewReader');
  15. goog.require('shaka.util.Error');
  16. goog.require('shaka.util.Mp4Parser');
  17. goog.require('shaka.util.Mp4BoxParsers');
  19. /**
  20.  * MPEG4 stream parser used for extracting 708 closed captions data.
  21.  * @implements {shaka.extern.ICeaParser}
  22.  * @export
  23.  */
  24. shaka.cea.Mp4CeaParser = class {
  25.   /** */
  26.   constructor() {
  27.     /**
  28.      * SEI data processor.
  29.      * @private
  30.      * @const {!shaka.cea.SeiProcessor}
  31.      */
  32.     this.seiProcessor_ = new shaka.cea.SeiProcessor();
  34.     /**
  35.      * Map of track id to corresponding timescale.
  36.      * @private {!Map<number, number>}
  37.      */
  38.     this.trackIdToTimescale_ = new Map();
  40.     /**
  41.      * Default sample duration, as specified by the TREX box.
  42.      * @private {number}
  43.      */
  44.     this.defaultSampleDuration_ = 0;
  46.     /**
  47.      * Default sample size, as specified by the TREX box.
  48.      * @private {number}
  49.      */
  50.     this.defaultSampleSize_ = 0;
  52.     /**
  53.      * @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
  54.      */
  55.     this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
  56.   }
  58.   /**
  59.    * Parses the init segment. Gets Default Sample Duration and Size from the
  60.    * TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
  61.    * contains a track header (TKHD) containing track ID, and a media header box
  62.    * (MDHD) containing the timescale for the track
  63.    * @override
  64.    */
  65.   init(initSegment) {
  66.     const Mp4Parser = shaka.util.Mp4Parser;
  67.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  68.     const trackIds = [];
  69.     const timescales = [];
  71.     const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
  73.     new Mp4Parser()
  74.         .box('moov', Mp4Parser.children)
  75.         .box('mvex', Mp4Parser.children)
  76.         .fullBox('trex', (box) => {
  77.           const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
  78.               box.reader);
  80.           this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
  81.           this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
  82.         })
  83.         .box('trak', Mp4Parser.children)
  84.         .fullBox('tkhd', (box) => {
  85.           goog.asserts.assert(
  86.               box.version != null,
  87.               'TKHD is a full box and should have a valid version.');
  88.           const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
  89.               box.reader, box.version);
  90.           trackIds.push(parsedTKHDBox.trackId);
  91.         })
  92.         .box('mdia', Mp4Parser.children)
  93.         .fullBox('mdhd', (box) => {
  94.           goog.asserts.assert(
  95.               box.version != null,
  96.               'MDHD is a full box and should have a valid version.');
  97.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  98.               box.reader, box.version);
  99.           timescales.push(parsedMDHDBox.timescale);
  100.         })
  101.         .box('minf', Mp4Parser.children)
  102.         .box('stbl', Mp4Parser.children)
  103.         .fullBox('stsd', Mp4Parser.sampleDescription)
  105.         // These are the various boxes that signal a codec.
  106.         .box('avc1', codecBoxParser)
  107.         .box('avc3', codecBoxParser)
  108.         .box('hev1', codecBoxParser)
  109.         .box('hvc1', codecBoxParser)
  110.         .box('dvav', codecBoxParser)
  111.         .box('dva1', codecBoxParser)
  112.         .box('dvh1', codecBoxParser)
  113.         .box('dvhe', codecBoxParser)
  115.         // This signals an encrypted sample, which we can go inside of to find
  116.         // the codec used.
  117.         .box('encv', Mp4Parser.visualSampleEntry)
  118.         .box('sinf', Mp4Parser.children)
  119.         .box('frma', (box) => {
  120.           const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  121.           this.setBitstreamFormat_(codec);
  122.         })
  124.         .parse(initSegment, /* partialOkay= */ true);
  126.     // At least one track should exist, and each track should have a
  127.     // corresponding Id in TKHD box, and timescale in its MDHD box
  128.     if (!trackIds.length|| !timescales.length ||
  129.       trackIds.length != timescales.length) {
  130.       throw new shaka.util.Error(
  131.           shaka.util.Error.Severity.CRITICAL,
  132.           shaka.util.Error.Category.TEXT,
  133.           shaka.util.Error.Code.INVALID_MP4_CEA);
  134.     }
  136.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  137.       shaka.log.alwaysWarn(
  138.           'Unable to determine bitstream format for CEA parsing!');
  139.     }
  141.     // Populate the map from track Id to timescale
  142.     trackIds.forEach((trackId, idx) => {
  143.       this.trackIdToTimescale_.set(trackId, timescales[idx]);
  144.     });
  145.   }
  147.   /**
  148.    * Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
  149.    * pairs. The following logic gets the necessary info from MOOFs to parse
  150.    * MDATs (base media decode time, sample sizes/offsets/durations, etc),
  151.    * and then parses the MDAT boxes for CEA-708 packets using this information.
  152.    * CEA-708 packets are returned in the callback.
  153.    * @override
  154.    */
  155.   parse(mediaSegment) {
  156.     const Mp4Parser = shaka.util.Mp4Parser;
  157.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  159.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  160.       // We don't know how to extract SEI from this.
  161.       return [];
  162.     }
  164.     /** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
  165.     const captionPackets = [];
  167.     let moofOffset = 0;
  169.     /** @type {!Array<!shaka.cea.Mp4CeaParser.ParsedTRAF>} */
  170.     let parsedTRAFs = [];
  172.     new Mp4Parser()
  173.         .box('moof', (box) => {
  174.           moofOffset = box.start;
  175.           // traf box parsing is reset on each moof.
  176.           parsedTRAFs = [];
  177.           Mp4Parser.children(box);
  178.         })
  179.         .box('traf', (box) => {
  180.           parsedTRAFs.push({
  181.             baseMediaDecodeTime: null,
  182.             defaultSampleDuration: this.defaultSampleDuration_,
  183.             defaultSampleSize: this.defaultSampleSize_,
  184.             parsedTRUNs: [],
  185.             timescale: shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE,
  186.           });
  187.           Mp4Parser.children(box);
  188.         })
  189.         .fullBox('trun', (box) => {
  190.           goog.asserts.assert(
  191.               box.version != null && box.flags != null,
  192.               'TRUN is a full box and should have a valid version & flags.');
  194.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  196.           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
  197.               box.reader, box.version, box.flags);
  198.           lastTRAF.parsedTRUNs.push(parsedTRUN);
  199.         })
  200.         .fullBox('tfhd', (box) => {
  201.           goog.asserts.assert(
  202.               box.flags != null,
  203.               'TFHD is a full box and should have valid flags.');
  205.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  207.           const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
  208.               box.reader, box.flags);
  210.           // If specified, defaultSampleDuration and defaultSampleSize
  211.           // override the ones specified in the TREX box
  212.           lastTRAF.defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
  213.               this.defaultSampleDuration_;
  215.           lastTRAF.defaultSampleSize = parsedTFHD.defaultSampleSize ||
  216.               this.defaultSampleSize_;
  218.           const trackId = parsedTFHD.trackId;
  220.           // Get the timescale from the track Id
  221.           if (this.trackIdToTimescale_.has(trackId)) {
  222.             lastTRAF.timescale = this.trackIdToTimescale_.get(trackId);
  223.           }
  224.         })
  225.         .fullBox('tfdt', (box) => {
  226.           goog.asserts.assert(
  227.               box.version != null,
  228.               'TFDT is a full box and should have a valid version.');
  230.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  232.           const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
  233.               box.reader, box.version);
  235.           lastTRAF.baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
  236.         })
  237.         .box('mdat', (box) => {
  238.           const offset = moofOffset - box.start - 8;
  239.           const initialPosition = box.reader.getPosition();
  240.           for (const parsedTRAF of parsedTRAFs) {
  241.             if (parsedTRAF.baseMediaDecodeTime === null) {
  242.               // This field should have been populated by the Base Media Decode
  243.               // Time in the tfdt box.
  244.               shaka.log.alwaysWarn(
  245.                   'Unable to find base media decode time for CEA captions!');
  246.               throw new shaka.util.Error(
  247.                   shaka.util.Error.Severity.CRITICAL,
  248.                   shaka.util.Error.Category.TEXT,
  249.                   shaka.util.Error.Code.INVALID_MP4_CEA);
  250.             }
  251.             box.reader.seek(initialPosition);
  252.             this.parseMdat_(box.reader,
  253.                 parsedTRAF.baseMediaDecodeTime,
  254.                 parsedTRAF.timescale,
  255.                 parsedTRAF.defaultSampleDuration,
  256.                 parsedTRAF.defaultSampleSize,
  257.                 offset,
  258.                 parsedTRAF.parsedTRUNs,
  259.                 captionPackets);
  260.           }
  261.         })
  262.         .parse(mediaSegment, /* partialOkay= */ false);
  264.     return captionPackets;
  265.   }
  267.   /**
  268.    * Parse MDAT box.
  269.    * @param {!shaka.util.DataViewReader} reader
  270.    * @param {number} time
  271.    * @param {number} timescale
  272.    * @param {number} defaultSampleDuration
  273.    * @param {number} defaultSampleSize
  274.    * @param {number} offset
  275.    * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  276.    * @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
  277.    * @private
  278.    */
  279.   parseMdat_(reader, time, timescale, defaultSampleDuration,
  280.       defaultSampleSize, offset, parsedTRUNs, captionPackets) {
  281.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  282.     const CeaUtils = shaka.cea.CeaUtils;
  283.     let sampleIndex = 0;
  285.     // The fields in each ParsedTRUNSample contained in the sampleData
  286.     // array are nullable. In the case of sample data and sample duration,
  287.     // we use the defaults provided by the TREX/TFHD boxes. For sample
  288.     // composition time offset, we default to 0.
  289.     let sampleSize = defaultSampleSize;
  291.     // Combine all sample data.  This assumes that the samples described across
  292.     // multiple trun boxes are still continuous in the mdat box.
  293.     const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
  294.     const sampleData = [].concat(...sampleDatas);
  296.     if (sampleData.length) {
  297.       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
  298.     }
  300.     reader.skip(offset + parsedTRUNs[0].dataOffset);
  302.     while (reader.hasMoreData()) {
  303.       const naluSize = reader.readUint32();
  304.       const naluHeader = reader.readUint8();
  305.       let naluType = null;
  306.       let isSeiMessage = false;
  307.       let naluHeaderSize = 1;
  309.       goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
  310.           'Bitstream format should have been checked before now!');
  311.       switch (this.bitstreamFormat_) {
  312.         case BitstreamFormat.H264:
  313.           naluType = naluHeader & 0x1f;
  314.           isSeiMessage = naluType == CeaUtils.H264_NALU_TYPE_SEI;
  315.           break;
  317.         case BitstreamFormat.H265:
  318.           naluHeaderSize = 2;
  319.           reader.skip(1);
  320.           naluType = (naluHeader >> 1) & 0x3f;
  321.           isSeiMessage =
  322.               naluType == CeaUtils.H265_PREFIX_NALU_TYPE_SEI ||
  323.               naluType == CeaUtils.H265_SUFFIX_NALU_TYPE_SEI;
  324.           break;
  326.         default:
  327.           return;
  328.       }
  330.       if (isSeiMessage) {
  331.         let timeOffset = 0;
  333.         if (sampleIndex < sampleData.length) {
  334.           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
  335.         }
  337.         const pts = (time + timeOffset) / timescale;
  338.         for (const packet of this.seiProcessor_
  339.             .process(reader.readBytes(naluSize - naluHeaderSize))) {
  340.           captionPackets.push({
  341.             packet,
  342.             pts,
  343.           });
  344.         }
  345.       } else {
  346.         try {
  347.           reader.skip(naluSize - naluHeaderSize);
  348.         } catch (e) {
  349.           // It is necessary to ignore this error because it can break the start
  350.           // of playback even if the user does not want to see the subtitles.
  351.           break;
  352.         }
  353.       }
  354.       sampleSize -= (naluSize + 4);
  355.       if (sampleSize == 0) {
  356.         if (sampleIndex < sampleData.length) {
  357.           time += sampleData[sampleIndex].sampleDuration ||
  358.               defaultSampleDuration;
  359.         } else {
  360.           time += defaultSampleDuration;
  361.         }
  363.         sampleIndex++;
  365.         if (sampleIndex < sampleData.length) {
  366.           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
  367.         } else {
  368.           sampleSize = defaultSampleSize;
  369.         }
  370.       }
  371.     }
  372.   }
  374.   /**
  375.    * @param {string} codec A fourcc for a codec.
  376.    * @private
  377.    */
  378.   setBitstreamFormat_(codec) {
  379.     if (codec in shaka.cea.Mp4CeaParser.CodecBitstreamMap_) {
  380.       this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.CodecBitstreamMap_[codec];
  381.     }
  382.   }
  383. };
  385. /** @enum {number} */
  386. shaka.cea.Mp4CeaParser.BitstreamFormat = {
  387.   UNKNOWN: 0,
  388.   H264: 1,
  389.   H265: 2,
  390. };
  392. /** @private {Object.<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
  393. shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = {
  394.   'avc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  395.   'avc3': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  396.   'hev1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  397.   'hvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  398.   // Dolby Vision based in AVC
  399.   'dvav': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  400.   'dva1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  401.   // Dolby Vision based in HEVC
  402.   'dvh1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  403.   'dvhe': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  404. };
  406. /**
  407.  * @typedef {{
  408.  *    baseMediaDecodeTime: ?number,
  409.  *    defaultSampleDuration: number,
  410.  *    defaultSampleSize: number,
  411.  *    parsedTRUNs: !Array<shaka.util.ParsedTRUNBox>,
  412.  *    timescale: number
  413.  * }}
  414.  *
  415.  * @property {?number} baseMediaDecodeTime
  416.  * @property {number} defaultSampleDuration
  417.  * @property {number} defaultSampleSize
  418.  * @property {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  419.  * @property {?number} timescale
  420.  */
  421. shaka.cea.Mp4CeaParser.ParsedTRAF;
  423. shaka.media.ClosedCaptionParser.registerParser('video/mp4',
  424.     () => new shaka.cea.Mp4CeaParser());