fis_lib_media_rt/wwwroot/js/video_processor.js

/**
 * 基于WebGL的视频裁切处理器
 * @param {Array} configs 配置数组，每个元素为一个配置对象，包含以下属性：
 *   - type: 裁切类型，默认值为"none"
 *   - x: 裁切区域的起始横坐标百分比（左上角），范围介于0到1之间，默认值为0
 *   - y: 裁切区域的起始纵坐标百分比（左上角），范围介于0到1之间，默认值为0
 *   - w: 裁切区域的宽度百分比，范围介于0到1之间，默认值为1
 *   - h: 裁切区域的高度百分比，范围介于0到1之间，默认值为1
 *
 * 使用方式：
 * 1. 创建一个VideoProcessor实例，传入适当的配置数组。
 * 2. 调用initializeById方法，传入源canvas的ID和目标canvas ID数组。
 *    这会初始化源和目标canvas，并将源canvas内容处理后绘制到目标canvas。
 * 3. 在需要更新帧时（例如，源视频播放新的一帧时），调用drawFrame方法。
 *    这会将当前源canvas中的帧复制并按照给定配置处理，然后绘制到目标canvas。
 *
 * 示例代码：
 * ```js
 * const configs = [
 *   { type: 'none', x: 0, y: 0, w: 1, h: 1 },
 *   { type: "scaleOnly", x: 2/3, y: 0.0, w: 1/3, h: 4/9 },
 *   { type: "threePart", x: 0.0, y: 22/27, w: 1, h: 4/27 },
 * ];
 * const videoProcessor = new VideoProcessor(configs);
 * videoProcessor.initializeById("video1", ["camera1", "camera2", "camera3"]);
 *
 * // 每当源视频播放新的一帧时执行以下操作
 * videoProcessor.drawFrame();
 * ```
 */
class VideoProcessor {
  constructor(configs = []) {
    this.configs = configs;
    this.vertexShaderSource = this._createVertexShaderSource();
  }

  initialize(sourceCanvas, targetCanvasArray) {
    this.canvasSource = sourceCanvas;
    this.canvasTargets = targetCanvasArray;

    this.glSource = this.canvasSource.getContext('webgl');

    // 初始化帧缓冲区和纹理数组
    this.frameBuffers = [];
    this.textures = [];
    this.programs = [];

    this.canvasTargets.forEach((targetCanvas, index) => {
      targetCanvas.width = this.canvasSource.width;
      targetCanvas.height = this.canvasSource.height;

      const config = this.configs[index] || {
        type: 'none',
        x: 0,
        y: 0,
        w: 1,
        h: 1,
      };
      this.fragmentShaderSources = ['' * this.configs.length];
      this.fragmentShaderSources[index] =
        this._createFragmentShaderSource(config);

      const glTarget = this.canvasTargets[index].getContext('webgl');
      // 创建并初始化帧缓冲区和纹理
      this._initWebGLCanvas(glTarget, index);
    });
  }

  initializeById(sourceCanvasId, targetCanvasIds) {
    const sourceCanvas = document.getElementById(sourceCanvasId);
    const targetCanvasArray = targetCanvasIds.map((id) =>
      document.getElementById(id),
    );
    this.initialize(sourceCanvas, targetCanvasArray);
  }

  drawFrame() {
    this.canvasTargets.forEach((_, index) => {
      const glTarget = this.canvasTargets[index].getContext('webgl');
      this._copyOnVideoFrame(glTarget, index);
    });
  }

  _initWebGLCanvas(glTarget, index) {
    console.log(`开始初始化 webgl-${index}`);

    // 创建纹理
    const texture = glTarget.createTexture();
    this.textures.push(texture);
    glTarget.bindTexture(glTarget.TEXTURE_2D, this.textures[index]);
    glTarget.texParameteri(
      glTarget.TEXTURE_2D,
      glTarget.TEXTURE_MIN_FILTER,
      glTarget.LINEAR,
    );
    glTarget.texParameteri(
      glTarget.TEXTURE_2D,
      glTarget.TEXTURE_WRAP_S,
      glTarget.CLAMP_TO_EDGE,
    );
    glTarget.texParameteri(
      glTarget.TEXTURE_2D,
      glTarget.TEXTURE_WRAP_T,
      glTarget.CLAMP_TO_EDGE,
    );
    glTarget.texImage2D(
      glTarget.TEXTURE_2D,
      0,
      glTarget.RGBA,
      this.canvasSource.width,
      this.canvasSource.height,
      0,
      glTarget.RGBA,
      glTarget.UNSIGNED_BYTE,
      null,
    );

    // 创建一个帧缓冲区
    const frameBuffer = glTarget.createFramebuffer();
    this.frameBuffers.push(frameBuffer);
    glTarget.bindFramebuffer(glTarget.FRAMEBUFFER, this.frameBuffers[index]);
    glTarget.framebufferTexture2D(
      glTarget.FRAMEBUFFER,
      glTarget.COLOR_ATTACHMENT0,
      glTarget.TEXTURE_2D,
      this.textures[index],
      0,
    );
    // 检查状态
    var status = glTarget.checkFramebufferStatus(glTarget.FRAMEBUFFER);
    if (status == glTarget.FRAMEBUFFER_COMPLETE) {
      console.log('Framebuffer is complete.');
    } else {
      console.error('Framebuffer is incomplete:', status);
    }
    // 创建着色器程序
    var vertexShader = this._createShader(
      glTarget,
      glTarget.VERTEX_SHADER,
      this.vertexShaderSource,
    );
    // 使用对应的片段着色器源创建并链接不同的着色器程序
    var fragmentShader = this._createShader(
      glTarget,
      glTarget.FRAGMENT_SHADER,
      this.fragmentShaderSources[index],
    );
    const program = glTarget.createProgram();
    this.programs.push(program);
    glTarget.attachShader(this.programs[index], vertexShader);
    glTarget.attachShader(this.programs[index], fragmentShader);
    glTarget.linkProgram(this.programs[index]);
    glTarget.useProgram(this.programs[index]);
    var success = glTarget.getProgramParameter(
      this.programs[index],
      glTarget.LINK_STATUS,
    );
    if (success) {
      console.log('Program linked successfully.');
    } else {
      console.error(glTarget.getProgramInfoLog(this.programs[index]));
      glTarget.deleteProgram(this.programs[index]);
    }

    // 创建顶点缓冲区，并设置 WebGL 的属性与通道（Channels）
    var positionLocation = glTarget.getAttribLocation(
      this.programs[index],
      'a_position',
    );
    var texCoordLocation = glTarget.getAttribLocation(
      this.programs[index],
      'a_texCoord',
    );

    var positionBuffer = glTarget.createBuffer();
    glTarget.bindBuffer(glTarget.ARRAY_BUFFER, positionBuffer);

    // 顶点坐标信息
    var positions = [-1, -1, 1, -1, -1, 1, 1, 1];

    glTarget.bufferData(
      glTarget.ARRAY_BUFFER,
      new Float32Array(positions),
      glTarget.STATIC_DRAW,
    );
    glTarget.enableVertexAttribArray(positionLocation);
    glTarget.vertexAttribPointer(
      positionLocation,
      2,
      glTarget.FLOAT,
      false,
      0,
      0,
    );

    var texCoordBuffer = glTarget.createBuffer();
    glTarget.bindBuffer(glTarget.ARRAY_BUFFER, texCoordBuffer);

    // 纹理坐标信息
    var texCoords = [0, 0, 1, 0, 0, 1, 1, 1];

    glTarget.bufferData(
      glTarget.ARRAY_BUFFER,
      new Float32Array(texCoords),
      glTarget.STATIC_DRAW,
    );
    glTarget.enableVertexAttribArray(texCoordLocation);
    glTarget.vertexAttribPointer(
      texCoordLocation,
      2,
      glTarget.FLOAT,
      false,
      0,
      0,
    );
    console.log(`初始化 webgl-${index} 完成`);
    console.log('--------------------');
  }

  _copyOnVideoFrame(glTarget, index) {
    var errorCode = glTarget.getError();
    if (errorCode !== glTarget.NO_ERROR) {
      console.error('WebGL error:', errorCode);
    }
    // 根据索引绑定相应的帧缓冲区
    glTarget.bindFramebuffer(glTarget.FRAMEBUFFER, this.frameBuffers[index]);
    glTarget.viewport(0, 0, this.canvasSource.width, this.canvasSource.height);

    // 从源canvas复制当前帧到纹理中
    glTarget.texImage2D(
      glTarget.TEXTURE_2D,
      0,
      glTarget.RGBA,
      glTarget.RGBA,
      glTarget.UNSIGNED_BYTE,
      this.canvasSource,
    );

    // 切换到目标canvas的帧缓冲区并绘制纹理
    glTarget.bindFramebuffer(glTarget.FRAMEBUFFER, null);
    glTarget.viewport(
      0,
      0,
      this.canvasTargets[index].width,
      this.canvasTargets[index].height,
    );
    this._drawTextureToTargetCanvas(glTarget, index); // 此处调用内部方法
    errorCode = glTarget.getError();
    if (errorCode !== glTarget.NO_ERROR) {
      console.error('WebGL error:', errorCode);
    }
  }

  _drawTextureToTargetCanvas(glTarget, index) {
    glTarget.clearColor(0, 0, 0, 1);
    glTarget.clear(glTarget.COLOR_BUFFER_BIT);

    glTarget.activeTexture(glTarget.TEXTURE0);
    glTarget.bindTexture(glTarget.TEXTURE_2D, this.textures[index]);
    glTarget.uniform1i(
      glTarget.getUniformLocation(this.programs[index], 'u_texture'),
      0,
    );

    glTarget.drawArrays(glTarget.TRIANGLE_STRIP, 0, 4);
  }

  _createShader(glTarget, type, source) {
    var shader = glTarget.createShader(type);
    glTarget.shaderSource(shader, source);
    glTarget.compileShader(shader);
    var success = glTarget.getShaderParameter(shader, glTarget.COMPILE_STATUS);
    if (success) {
      return shader;
    }
    console.error(glTarget.getShaderInfoLog(shader));
    glTarget.deleteShader(shader);
  }

  _createVertexShaderSource() {
    return `
    attribute vec4 a_position;
    attribute vec2 a_texCoord;
    varying vec2 v_texCoord;
    void main() {
      gl_Position = a_position;
      v_texCoord = a_texCoord;
    }
    `;
  }

  /**
   * 裁图模式
   * [scaleOnly: 只缩放，不裁剪]
   * [threePart: 三等分切割]
   * [camera720: 两等分切割]
   */
  _createFragmentShaderSource(config) {
    const { type, x, y, w, h } = config;
    const width = this._formatShaderNumber(w);
    const height = this._formatShaderNumber(h);
    const offsetX = this._formatShaderNumber(x);
    const offsetY = this._formatShaderNumber(y);
    switch (type) {
      case 'scaleOnly':
        return `
        precision mediump float;
        varying vec2 v_texCoord;
        uniform sampler2D u_texture;
        void main() {
          vec2 scaledTexCoord = vec2(v_texCoord.x * ${width}, v_texCoord.y * ${height});
          scaledTexCoord.x += ${offsetX};
          scaledTexCoord.y += ${offsetY};
          gl_FragColor = texture2D(u_texture, scaledTexCoord);
        }
        `;
      case 'threePart':
        return `
        precision mediump float;
        varying vec2 v_texCoord;
        uniform sampler2D u_texture;

        void main() {
          if (v_texCoord.y < (1.0 / 3.0)) {
            vec2 newTexCoord = vec2(v_texCoord.x / 3.0, v_texCoord.y * 3.0 * ${height} + ${offsetY});
            gl_FragColor = texture2D(u_texture, newTexCoord);
          } else if (v_texCoord.y < (2.0 / 3.0)) {
            vec2 newTexCoord = vec2(v_texCoord.x / 3.0 + 1.0 / 3.0, v_texCoord.y * 3.0 * ${height} + ${offsetY} - ${height});
            gl_FragColor = texture2D(u_texture, newTexCoord);
          } else {
            vec2 newTexCoord = vec2(v_texCoord.x / 3.0 + 2.0 / 3.0, v_texCoord.y * 3.0 * ${height} + ${offsetY} - 2.0 * ${height});
            gl_FragColor = texture2D(u_texture, newTexCoord);
          }
        }
        `;
      case "camera720":
        // + 1.0 / 720.0 解决拼接黑线问题
        return `
          precision mediump float;
          varying vec2 v_texCoord;
          uniform sampler2D u_texture;
  
          void main() {
            if (v_texCoord.y < (1.0 / 2.0)) {
              vec2 newTexCoord = vec2(v_texCoord.x / 2.0, v_texCoord.y * 2.0 * ${height} + ${offsetY});
              gl_FragColor = texture2D(u_texture, newTexCoord);
            } else {
              vec2 newTexCoord = vec2(v_texCoord.x / 2.0 + 1.0 / 2.0, v_texCoord.y * 2.0 * ${height} + ${offsetY} - 1.0 * ${height} + 1.0 / 720.0 );
              gl_FragColor = texture2D(u_texture, newTexCoord);
            }
          }
          `;
      case "oldTwoPart":
        return `
          precision mediump float;
          varying vec2 v_texCoord;
          uniform sampler2D u_texture;
          void main() {
            vec2 newTexCoord = vec2(v_texCoord.x / 2.0, v_texCoord.y * ${height} + ${offsetY});
            gl_FragColor = texture2D(u_texture, newTexCoord);
          }
          `;
      case "oldThreePart":
        return `
          precision mediump float;
          varying vec2 v_texCoord;
          uniform sampler2D u_texture;
          void main() {
            vec2 newTexCoord = vec2(v_texCoord.x / 3.0, v_texCoord.y * ${height} + ${offsetY});
            gl_FragColor = texture2D(u_texture, newTexCoord);
          }
          `;
      default:
        return `
        precision mediump float;
        varying vec2 v_texCoord;
        uniform sampler2D u_texture;
        void main() {
          gl_FragColor = texture2D(u_texture, v_texCoord);
        }
        `;
    }
  }

  _formatShaderNumber(number) {
    const clampedNumber = Math.max(0, Math.min(1, number));
    return clampedNumber.toFixed(6);
  }
}