From 333aa66f0f59f8e859fcee5b8678e86bae061520 Mon Sep 17 00:00:00 2001 From: rsonghuster Date: Thu, 22 Aug 2024 14:00:44 +0800 Subject: [PATCH] change multimedia-process-flow-v3 to v3 --- .github/workflows/registry-publish.yml | 34 ++-- auto-publish.py | 37 ++++ readme.md | 3 - update.list | 1 - video-process-flow/hook/index.js | 15 +- video-process-flow/publish.yaml | 75 ++++--- video-process-flow/readme.md | 126 +++++++++++- .../src/code/flows/input-fc.json | 7 - .../src/code/flows/video-processing-fc.yml | 72 ------- video-process-flow/src/code/merge/index.py | 8 +- video-process-flow/src/code/split/index.py | 33 +++- .../src/code/transcode/index.py | 2 +- .../src/code/workflows/fc-input.json | 12 ++ .../workflows/multimedia-process-workflow.yml | 117 +++++++++++ video-process-flow/src/readme.md | 99 ++++------ video-process-flow/src/s.yaml | 185 ++++++++---------- 16 files changed, 505 insertions(+), 321 deletions(-) create mode 100644 auto-publish.py delete mode 100644 update.list delete mode 100644 video-process-flow/src/code/flows/input-fc.json delete mode 100644 video-process-flow/src/code/flows/video-processing-fc.yml create mode 100644 video-process-flow/src/code/workflows/fc-input.json create mode 100644 video-process-flow/src/code/workflows/multimedia-process-workflow.yml diff --git a/.github/workflows/registry-publish.yml b/.github/workflows/registry-publish.yml index 6c1d746..d8c233a 100644 --- a/.github/workflows/registry-publish.yml +++ b/.github/workflows/registry-publish.yml @@ -1,29 +1,27 @@ -name: publish package to serverless-hub +name: auto try publish all on: - release: - types: [created] + push: + branches: + - V3 +env: + REGISTRY_TOKEN: ${{secrets.ALIBABA_REGISTRY_V3_PUBLISH_TOKEN}} jobs: - deploy: + auto-publish: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/setup-node@v2 with: - python-version: '3.x' - - uses: actions/setup-node@v1 + node-version: 16 + registry-url: https://registry.npmjs.org/ + - uses: actions/setup-python@v5 with: - node-version: 12 - - name: Install dependencies + python-version: '3.10' + - name: install s run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - pip install requests - - name: Publish package - env: - publish_token: ${{ secrets.alibaba_registry_publish_token }} + npm i @serverless-devs/s -g + - name: publish run: | - ls - python publish.py \ No newline at end of file + python auto-publish.py \ No newline at end of file diff --git a/auto-publish.py b/auto-publish.py new file mode 100644 index 0000000..d363964 --- /dev/null +++ b/auto-publish.py @@ -0,0 +1,37 @@ +# coding=utf-8 + +import os +import subprocess + +os.system("s registry login --token {}".format(os.environ["REGISTRY_TOKEN"])) + + +def search_publish_yaml(directory): + succ_app_list = [] + for root, dirs, files in os.walk(directory): + if "publish.yaml" in files: + d = os.path.join(os.getcwd(), root) + d = d.replace("/./", "/") + cmd = ["s", "registry", "publish"] + result = subprocess.run( + cmd, cwd=d, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + exit_code = result.returncode + print("publish {}; =====> ExitCode={}".format(d, exit_code)) + + if exit_code != 0: + errMsg = "" + errMsg += result.stdout + errMsg += result.stderr + if "当前版本号应当大于前一次版本号" not in errMsg: + print(errMsg) + else: + succ_app_list.append(d) + + print("\n\npublish success app list:\n") + for item in succ_app_list: + print("{} published success".format(item)) + + +# 在当前目录递归搜索 +search_publish_yaml(".") diff --git a/readme.md b/readme.md index 5784514..863fe65 100644 --- a/readme.md +++ b/readme.md @@ -12,7 +12,4 @@ - GetSprites: 功能强大雪碧图制作函数 ([案例代码](./ffmpeg-app/src/functions/get_sprites)) - VideoWatermark: 功能强大的视频添加水印功能 ([案例代码](./ffmpeg-app/src/functions/video_watermark)) - [基于 FFmpeg 实现音视频转码](./transcode/src): `s init video-transcode` -- [基于 FFmpeg 实现 HTTP 触发器触发音视频转码](./http-transcode/src): `s init http-video-transcode` - [基于 FC + Serverless Workflow + OSS + NAS + FFmpeg 实现的弹性高可用、并行处理的视频转码服务](./video-process-flow/src): `s init video-process-flow` -- [对直播视频流截图的应用](./rtmp-snapshot/src): `s init rtmp-snapshot` -- [一个对浏览器全景录制](./headless-ffmpeg/src): `s init headless-ffmpeg` diff --git a/update.list b/update.list deleted file mode 100644 index ec0910a..0000000 --- a/update.list +++ /dev/null @@ -1 +0,0 @@ -./video-process-flow \ No newline at end of file diff --git a/video-process-flow/hook/index.js b/video-process-flow/hook/index.js index dc925a9..10570e9 100644 --- a/video-process-flow/hook/index.js +++ b/video-process-flow/hook/index.js @@ -3,23 +3,20 @@ async function preInit(inputObj) { } async function postInit(inputObj) { - console.log(`\n _______ _______ __ __ _______ _______ _______ + console.log(`\n _______ _______ __ __ _______ _______ _______ | || || |_| || || || | | ___|| ___|| || _ || ___|| ___| - | |___ | |___ | || |_| || |___ | | __ + | |___ | |___ | || |_| || |___ | | __ | ___|| ___|| || ___|| ___|| || | | | | | | ||_|| || | | |___ | |_| | |___| |___| |_| |_||___| |_______||_______| `) - console.log(`\n Welcome to the ffmpeg-app application - This application requires to open these services: + console.log(`\n Welcome to the multimedia-process-flow-v3 application + This application requires to open these services: FC : https://fc.console.aliyun.com/ - This application can help you quickly deploy the ffmpeg-app project. - The application uses FC component：https://github.com/devsapp/fc - The application homepage: https://github.com/devsapp/start-ffmpeg\n`) + This application can help you quickly deploy the ffmpeg project. + The application uses FC component：https://docs.serverless-devs.com/user-guide/aliyun/#fc3\n`) - const { artTemplate } = inputObj; - artTemplate("code/flows/video-processing-fc.yml"); } module.exports = { diff --git a/video-process-flow/publish.yaml b/video-process-flow/publish.yaml index 26d1dbf..0e47f36 100644 --- a/video-process-flow/publish.yaml +++ b/video-process-flow/publish.yaml @@ -1,10 +1,12 @@ -Type: Application -Name: video-process-flow -Version: 0.1.6 +Edition: 3.0.0 +Type: Project +Name: multimedia-process-flow-v3 +Version: 0.0.3 Provider: - 阿里云 Description: 基于 FC + Serverless Workflow + OSS + NAS + FFmpeg 实现的弹性高可用、并行处理的视频转码服务 -HomePage: https://github.com/devsapp/start-ffmpeg/tree/master/video-process-flow +HomePage: https://github.com/devsapp/start-ffmpeg/tree/master/multimedia-process-flow +Effective: Public Tags: - flow - ffmpeg @@ -18,19 +20,16 @@ Service: - AliyunFCFullAccess 硬盘挂载: Authorities: - - AliyunNASFullAccess - VPC: + - AliyunFCServerlessDevsRolePolicy + 专有网络: Authorities: - - AliyunVPCFullAccess - OSS: + - AliyunFCServerlessDevsRolePolicy + 对象存储: Authorities: - - AliyunOSSFullAccess - 工作流: + - AliyunFCServerlessDevsRolePolicy + 云工作流: Authorities: - AliyunFnFFullAccess - 其它: - Authorities: - - AliyunECSFullAccess Parameters: type: object additionalProperties: false # 不允许增加其他属性 @@ -57,14 +56,29 @@ Parameters: - cn-shenzhen - ap-southeast-1 - us-west-1 - serviceName: - title: 服务名 + + flowName: + title: 工作流程名称 + type: string + default: multimedia-process-flow-${default-suffix} + description: Serverless 工作流流程名称 + + fnfRoleArn: + title: 工作流 RAM角色ARN type: string - default: video-process-flow-${default-suffix} - pattern: "^[a-zA-Z_][a-zA-Z0-9-_]{0,127}$" - description: 应用所属的函数计算服务 - serviceRoleArn: - title: 函数计算Service RAM角色ARN + default: "" + pattern: "^acs:ram::[0-9]*:role/.*$" + description: 应用所属的工作流需要的 role, 请提前创建好对应的 role, 授信工作流服务, 并配置好 AliyunFCInvocationAccess 和 AliyunFnFFullAccess policy。 + required: true + x-role: + name: fnf-execution-default-role + service: FNF + authorities: + - AliyunFCInvocationAccess + - AliyunFnFFullAccess + + functionRoleArn: + title: 函数角色 type: string default: "" pattern: "^acs:ram::[0-9]*:role/.*$" @@ -79,6 +93,7 @@ Parameters: - AliyunOSSFullAccess - AliyunFCDefaultRolePolicy - AliyunFnFFullAccess + ossBucket: title: 对象存储存储桶名 type: string @@ -124,23 +139,3 @@ Parameters: type: string default: mp4, flv, avi description: 转码后的视频格式，如果有需要输出多种格式， 使用逗号分隔 - - flowName: - title: 工作流程名称 - type: string - default: video-process-flow - description: Serverless 工作流流程名称 - - fnfRoleArn: - title: 工作流 RAM角色ARN - type: string - default: "" - pattern: "^acs:ram::[0-9]*:role/.*$" - description: 应用所属的工作流需要的 role, 请提前创建好对应的 role, 授信工作流服务, 并配置好 AliyunFCInvocationAccess 和 AliyunFnFFullAccess policy。 - required: true - x-role: - name: fnf-execution-default-role - service: FNF - authorities: - - AliyunFCInvocationAccess - - AliyunFnFFullAccess diff --git a/video-process-flow/readme.md b/video-process-flow/readme.md index 1105835..825e53a 120000 --- a/video-process-flow/readme.md +++ b/video-process-flow/readme.md @@ -1 +1,125 @@ -src/readme.md \ No newline at end of file + +> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ${模版名称}` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 + +# multimedia-process-flow-v3 帮助文档 + + + +基于 FC + Serverless Workflow + OSS + NAS + FFmpeg 实现的弹性高可用、并行处理的视频转码服务 + + + + + + + + + + + + + + + +## 前期准备 + +使用该项目，您需要有开通以下服务并拥有对应权限： + + + + + +| 服务/业务 | 权限 | 相关文档 | +| --- | --- | --- | +| 函数计算 | AliyunFCFullAccess | [帮助文档](https://help.aliyun.com/product/2508973.html) [计费文档](https://help.aliyun.com/document_detail/2512928.html) | +| 硬盘挂载 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/nas) [计费文档](https://help.aliyun.com/zh/nas/product-overview/billing) | +| 专有网络 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/vpc) [计费文档](https://help.aliyun.com/zh/vpc/product-overview/billing) | +| 对象存储 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/oss) [计费文档](https://help.aliyun.com/zh/oss/product-overview/billing) | +| 云工作流 | AliyunFnFFullAccess | [帮助文档](undefined) [计费文档](undefined) | + + + + + + + + + + + + + + + +## 部署 & 体验 + + + +- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=multimedia-process-flow-v3) ， + [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=multimedia-process-flow-v3) 该应用。 + + + + +- 通过 [Serverless Devs Cli](https://docs.serverless-devs.com/user-guide/install) 进行部署： + - [安装 Serverless Devs Cli 开发者工具](https://docs.serverless-devs.com/user-guide/install) ，并进行[授权信息配置]( https://docs.serverless-devs.com/user-guide/config) ； + - 初始化项目：`s init multimedia-process-flow-v3 -d multimedia-process-flow-v3` + - 进入项目，并进行项目部署：`cd multimedia-process-flow-v3 && s deploy -y` + + + +## 案例介绍 + + + +本案例将FFmpeg包装为一款音视频处理工作流，十分适用于加快视频转码处理速度的场景， 基本原理是对每一个视频，先进行切片处理，然后并行转码切片，最后合成，通过设置合理的切片时间，可以大大加速较大视频的转码速度。 + +FFmpeg 是音视频处理领域的一款强大工具，它可以被用于格式转换、编解码、录制和流处理等多种音视频相关任务。该技术因其高效性和多功能性，在视频处理、直播流媒体、数字媒体播放和编辑等多个领域中得到广泛应用。截至2023年4月，FFmpeg 在 GitHub 上的 star 数接近30,000，这反映了其在开源社区的受欢迎程度和影响力。被Netflix、YouTube、Facebook等多家知名企业采用，这些公司依赖 FFmpeg 处理其庞大的视频数据，以提供流畅的媒体播放体验、视频内容编辑和优化视频传输效率等服务。 + + + +## 使用流程 + + + +上传一个 mov 格式的视频到对象存储 bucket, OSS 触发器自动触发函数执行，函数调用工作流执行，工作流会同时进行 1 种或者多种格式的转码(由 s.yaml 中的 DST_FORMATS 参数控制)， 本示例默认配置的是同时进行 mp4, flv, avi 格式的转码。 + +基于该工作流，您可以实现如下需求: + +1 一个视频文件可以同时被转码成各种格式以及其他各种自定义处理，比如增加水印处理或者在 after-process 更新信息到数据库等。 + +2 当有多个文件同时上传到 OSS，函数计算会自动伸缩， 并行处理多个文件， 同时每次文件转码成多种格式也是并行。 + +3 结合 NAS + 视频切片， 可以解决超大视频的转码， 对于每一个视频，先进行切片处理，然后并行转码切片，最后合成，通过设置合理的切片时间，可以大大加速较大视频的转码速度。 + +![image](https://img.alicdn.com/tfs/TB1A.PSzrj1gK0jSZFuXXcrHpXa-570-613.png) + + +**操作视频教程:** + +[![Watch the video](https://img.alicdn.com/imgextra/i2/O1CN01XvnqJu1XLS8SAU7LT_!!6000000002907-2-tps-250-155.png)](http://devsapp.functioncompute.com/video/multimedia-process-flow.mp4) + +**P.S.** 当您想要仅在一个简单的函数中直接完成视频处理逻辑时，可以参考[音视频转码 Job](https://github.com/devsapp/start-ffmpeg/tree/V3/transcode/src) + + + +## 注意事项 + + + + + + + + +## 开发者社区 + +您如果有关于错误的反馈或者未来的期待，您可以在 [Serverless Devs repo Issues](https://github.com/serverless-devs/serverless-devs/issues) 中进行反馈和交流。如果您想要加入我们的讨论组或者了解 FC 组件的最新动态，您可以通过以下渠道进行： + +

+ +| | | | +| --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +|

微信公众号：`serverless`
|
微信小助手：`xiaojiangwh`
|
钉钉交流群：`33947367`
| +

+ diff --git a/video-process-flow/src/code/flows/input-fc.json b/video-process-flow/src/code/flows/input-fc.json deleted file mode 100644 index 6cc5e2f..0000000 --- a/video-process-flow/src/code/flows/input-fc.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "oss_bucket_name": "fnf-test", - "video_key": "fnf_video/inputs/fc-official-short.mov", - "output_prefix": "fnf_video/outputs/fc/1", - "segment_time_seconds": 15, - "dst_formats": ["mp4", "flv"] -} \ No newline at end of file diff --git a/video-process-flow/src/code/flows/video-processing-fc.yml b/video-process-flow/src/code/flows/video-processing-fc.yml deleted file mode 100644 index c740d43..0000000 --- a/video-process-flow/src/code/flows/video-processing-fc.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: v1beta1 -type: flow -steps: - - type: task - name: Split - resourceArn: 'acs:fc:::services/{{serviceName}}/functions/split' - retry: - - errors: - - FC.ResourceThrottled - - FC.ResourceExhausted - - FC.InternalServerError - - FnF.TaskTimeout - - FC.Unknown - intervalSeconds: 3 - maxAttempts: 16 - multiplier: 2 - - type: foreach - name: ParallelTranscode - iterationMapping: - collection: $.dst_formats - index: index - item: target_type - steps: - - type: foreach - name: Transcode_splits - iterationMapping: - collection: $.split_keys - index: index - item: split_video_key - steps: - - type: task - name: Transcode - resourceArn: 'acs:fc:::services/{{serviceName}}/functions/transcode' - retry: - - errors: - - FC.ResourceThrottled - - FC.ResourceExhausted - - FC.InternalServerError - - FnF.TaskTimeout - - FC.Unknown - intervalSeconds: 3 - maxAttempts: 16 - multiplier: 2 - - type: task - name: Merge - resourceArn: 'acs:fc:::services/{{serviceName}}/functions/merge' - retry: - - errors: - - FC.ResourceThrottled - - FC.ResourceExhausted - - FC.InternalServerError - - FnF.TaskTimeout - - FC.Unknown - intervalSeconds: 3 - maxAttempts: 16 - multiplier: 2 - outputMappings: - - target: video_proc_dir - source: $input.video_proc_dir - - type: task - name: after-process - resourceArn: 'acs:fc:::services/{{serviceName}}/functions/after-process' - retry: - - errors: - - FC.ResourceThrottled - - FC.ResourceExhausted - - FC.InternalServerError - - FnF.TaskTimeout - - FC.Unknown - intervalSeconds: 3 - maxAttempts: 16 - multiplier: 2 \ No newline at end of file diff --git a/video-process-flow/src/code/merge/index.py b/video-process-flow/src/code/merge/index.py index 943a73b..b4fbcf2 100644 --- a/video-process-flow/src/code/merge/index.py +++ b/video-process-flow/src/code/merge/index.py @@ -59,10 +59,10 @@ def handler(event, context): output_prefix = evt['output_prefix'] video_type = evt['target_type'] video_process_dir = evt['video_proc_dir'] - + transcoded_split_keys = [] for k in split_keys: - fileDir, shortname, extension = get_fileNameExt(k) + fileDir, shortname, extension = get_fileNameExt(k['filekey']) transcoded_filename = 'transcoded_%s.%s' % (shortname, video_type) transcoded_filepath = os.path.join(fileDir, transcoded_filename) transcoded_split_keys.append(transcoded_filepath) @@ -75,12 +75,12 @@ def handler(event, context): if len(transcoded_split_keys) == 0: raise Exception("no transcoded_split_keys") - + LOGGER.info({ "target_type": video_type, "transcoded_split_keys": transcoded_split_keys }) - + _, shortname, extension = get_fileNameExt(video_key) segs_filename = 'segs_%s.txt' % (shortname + video_type) segs_filepath = os.path.join(video_process_dir, segs_filename) diff --git a/video-process-flow/src/code/split/index.py b/video-process-flow/src/code/split/index.py index db708ad..686e55f 100644 --- a/video-process-flow/src/code/split/index.py +++ b/video-process-flow/src/code/split/index.py @@ -14,7 +14,7 @@ MAX_SPLIT_NUM = 100 -NAS_ROOT = "/mnt/auto/" +NAS_ROOT = "/mnt/{}/".format(os.environ['FC_FUNCTION_NAME']) class FFmpegError(Exception): def __init__(self, message, status): @@ -65,7 +65,8 @@ def handler(event, context): video_key = evt['video_key'] oss_bucket_name = evt['oss_bucket_name'] segment_time_seconds = str(evt['segment_time_seconds']) - + dst_formats = evt['dst_formats'] + shortname, extension = get_fileNameExt(video_key) video_name = shortname + extension @@ -95,9 +96,31 @@ def handler(event, context): for filename in os.listdir(video_proc_dir): if filename.startswith('split_' + shortname): filekey = os.path.join(video_proc_dir, filename) - split_keys.append(filekey) + split_keys.append( + { + "filekey": filekey + }) + + new_dst_formats = [] + + for df in dst_formats: + + new_sks = [] + for sk in split_keys: + new_sks.append({ + "filekey": sk["filekey"], + "video_proc_dir": video_proc_dir, + "format": df + }) + + new_dst_formats.append( + { + "format": df, + "video_proc_dir": video_proc_dir, + "split_keys": new_sks + }) return { - "split_keys": split_keys, - "video_proc_dir": video_proc_dir + "video_proc_dir": video_proc_dir, + "dst_formats": new_dst_formats } diff --git a/video-process-flow/src/code/transcode/index.py b/video-process-flow/src/code/transcode/index.py index 5a6af3b..38e5285 100644 --- a/video-process-flow/src/code/transcode/index.py +++ b/video-process-flow/src/code/transcode/index.py @@ -58,5 +58,5 @@ def handler(event, context): os.remove(transcoded_filepath) exec_FFmpeg_cmd(['ffmpeg', '-y', '-i', input_path, transcoded_filepath]) - return {} + return evt diff --git a/video-process-flow/src/code/workflows/fc-input.json b/video-process-flow/src/code/workflows/fc-input.json new file mode 100644 index 0000000..7aa5314 --- /dev/null +++ b/video-process-flow/src/code/workflows/fc-input.json @@ -0,0 +1,12 @@ +{ + "oss_bucket_name": "tt0126", + "video_key": "source/fnf-test.mov", + "output_prefix": "outputs", + "segment_time_seconds": 15, + "dst_formats": [ + "mp4", + "flv" + ] +} + +// oss-trigger-workflow 设置 oss触发器，设置前缀为 source，避免无限循环 \ No newline at end of file diff --git a/video-process-flow/src/code/workflows/multimedia-process-workflow.yml b/video-process-flow/src/code/workflows/multimedia-process-workflow.yml new file mode 100644 index 0000000..6cefd91 --- /dev/null +++ b/video-process-flow/src/code/workflows/multimedia-process-workflow.yml @@ -0,0 +1,117 @@ +Type: StateMachine +Name: ${vars.flowName} +SpecVersion: v1 +Description: Serverless workflow video processing +StartAt: Split +States: + - Action: FC:InvokeFunction + Type: Task + TaskMode: RequestComplete + Name: Split + Parameters: + resourceArn: acs:fc:::functions/${vars.flowName}_split + body.$: $Input + Next: ParallelTranscode + OutputConstructor: + split_keys.$: $Output.Body.split_keys + video_proc_dir.$: $Output.Body.video_proc_dir + dst_formats.$: $Output.Body.dst_formats + Retry: + - Errors: + - FC.ResourceThrottled + - FC.ResourceExhausted + - FC.InternalServerError + - FnF.TaskTimeout + - FC.Unknown + IntervalSeconds: 3 + MaxAttempts: 16 + BackoffRate: 2 + - Type: Map + Name: ParallelTranscode + ItemsPath: $Input.dst_formats + Next: after-process + InputConstructor: + split_keys.$: $Input.split_keys + video_proc_dir.$: $Input.video_proc_dir + dst_formats.$: $Input.dst_formats + OutputConstructor: + video_proc_dir.$: $Input.video_proc_dir + Processor: + StartAt: Transcode_splits + States: + - ItemsPath: $Input.split_keys + OutputConstructor: + Items.$: $Output.Items + format.$: $Input.format + split_keys.$: $Input.split_keys + video_proc_dir.$: $Input.video_proc_dir + Name: Transcode_splits + Next: Merge + Processor: + StartAt: Transcode + States: + - Action: FC:InvokeFunction + End: true + TaskMode: RequestComplete + Name: Transcode + Parameters: + resourceArn: acs:fc:::functions/${vars.flowName}_transcode + body: + split_video_key.$: $Input.filekey + target_type.$: $Input.format + video_proc_dir.$: $Input.video_proc_dir + Retry: + - Errors: + - FC.ResourceThrottled + - FC.ResourceExhausted + - FC.InternalServerError + - FnF.TaskTimeout + - FC.Unknown + IntervalSeconds: 3 + MaxAttempts: 16 + BackoffRate: 2 + Type: Task + Type: Map + - Action: FC:InvokeFunction + End: true + Name: Merge + TaskMode: RequestComplete + InputConstructor: + video_key.$: $Context.Execution.Input.video_key + oss_bucket_name.$: $Context.Execution.Input.oss_bucket_name + split_keys.$: $Input.split_keys + output_prefix.$: $Context.Execution.Input.output_prefix + target_type.$: $Input.format + video_proc_dir.$: $Input.video_proc_dir + Parameters: + resourceArn: acs:fc:::functions/${vars.flowName}_merge + body.$: $Input + Retry: + - Errors: + - FC.ResourceThrottled + - FC.ResourceExhausted + - FC.InternalServerError + - FnF.TaskTimeout + - FC.Unknown + IntervalSeconds: 3 + MaxAttempts: 16 + BackoffRate: 2 + Type: Task + - Type: Task + Action: FC:InvokeFunction + TaskMode: RequestComplete + End: true + Name: after-process + Parameters: + resourceArn: acs:fc:::functions/${vars.flowName}_after-process + body.$: $Input + Retry: + - Errors: + - FC.ResourceThrottled + - FC.ResourceExhausted + - FC.InternalServerError + - FnF.TaskTimeout + - FC.Unknown + IntervalSeconds: 3 + MaxAttempts: 16 + BackoffRate: 2 diff --git a/video-process-flow/src/readme.md b/video-process-flow/src/readme.md index f67e4a9..825e53a 100644 --- a/video-process-flow/src/readme.md +++ b/video-process-flow/src/readme.md @@ -1,18 +1,7 @@ -> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、服务名、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 - -# video-process-flow 帮助文档 -

- - - - - - - - - -

+> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ${模版名称}` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 + +# multimedia-process-flow-v3 帮助文档 @@ -22,7 +11,7 @@ -- [:smiley_cat: 代码](https://github.com/devsapp/start-ffmpeg/tree/master/video-process-flow/src) + @@ -34,38 +23,22 @@ ## 前期准备 -使用该项目，您需要有开通以下服务： +使用该项目，您需要有开通以下服务并拥有对应权限： -| 服务 | 备注 | -| --- | --- | -| 函数计算 FC | 转码等函数部署在函数计算 | -| Serverless 工作流 | 视频处理工作流部署在 Serverless 工作流 | -| 对象存储 OSS | 原视频位于 OSS | -| 文件存储 NAS | 视频临时处理工作区间位于文件存储 NAS | -| 专有网络 VPC | NAS 挂载点需要有 VPC | +| 服务/业务 | 权限 | 相关文档 | +| --- | --- | --- | +| 函数计算 | AliyunFCFullAccess | [帮助文档](https://help.aliyun.com/product/2508973.html) [计费文档](https://help.aliyun.com/document_detail/2512928.html) | +| 硬盘挂载 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/nas) [计费文档](https://help.aliyun.com/zh/nas/product-overview/billing) | +| 专有网络 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/vpc) [计费文档](https://help.aliyun.com/zh/vpc/product-overview/billing) | +| 对象存储 | AliyunFCServerlessDevsRolePolicy | [帮助文档](https://help.aliyun.com/zh/oss) [计费文档](https://help.aliyun.com/zh/oss/product-overview/billing) | +| 云工作流 | AliyunFnFFullAccess | [帮助文档](undefined) [计费文档](undefined) | -推荐您拥有以下的产品权限 / 策略： - - - - -| 服务/业务 | 权限 | 备注 | -| --- | --- | --- | -| 函数计算 | AliyunFCFullAccess | 创建或者更新转码等函数 | -| 硬盘挂载 | AliyunNASFullAccess | 视频临时处理工作区间位于文件存储 NAS， 需要有自动创建 NAS 的权限 | -| VPC | AliyunVPCFullAccess | NAS 需要 VPC 挂载点， 需要有 VPC 自动创建的能力 | -| OSS | AliyunOSSFullAccess | 创建 OSS 触发器需要的调用 OSS 相关 API 的权限 | -| 工作流 | AliyunFnFFullAccess | 创建或者更新音视频处理工作流 | -| 其它 | AliyunECSFullAccess | 函数计算 NAS 挂载点需要交换机和安全组， 需要有自动创建的权限 | - - - @@ -82,49 +55,59 @@ -- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=video-process-flow) ， - [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=video-process-flow) 该应用。 +- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=multimedia-process-flow-v3) ， + [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=multimedia-process-flow-v3) 该应用。 -- 通过 [Serverless Devs Cli](https://www.serverless-devs.com/serverless-devs/install) 进行部署： - - [安装 Serverless Devs Cli 开发者工具](https://www.serverless-devs.com/serverless-devs/install) ，并进行[授权信息配置](https://docs.serverless-devs.com/fc/config) ； - - 初始化项目：`s init video-process-flow -d video-process-flow ` - - 进入项目，并进行项目部署：`cd video-process-flow && s deploy - y` +- 通过 [Serverless Devs Cli](https://docs.serverless-devs.com/user-guide/install) 进行部署： + - [安装 Serverless Devs Cli 开发者工具](https://docs.serverless-devs.com/user-guide/install) ，并进行[授权信息配置]( https://docs.serverless-devs.com/user-guide/config) ； + - 初始化项目：`s init multimedia-process-flow-v3 -d multimedia-process-flow-v3` + - 进入项目，并进行项目部署：`cd multimedia-process-flow-v3 && s deploy -y` -## 应用详情 +## 案例介绍 -如下图所示， 假设用户上传一个 mov 格式的视频到 OSS, OSS 触发器自动触发函数执行， 函数调用 FnF 执行，FnF 同时进行 1 种或者多种格式的转码(由 s.yaml 中的 DST_FORMATS 参数控制)， 本示例配置的是同时进行 mp4, flv, avi 格式的转码。 +本案例将FFmpeg包装为一款音视频处理工作流，十分适用于加快视频转码处理速度的场景， 基本原理是对每一个视频，先进行切片处理，然后并行转码切片，最后合成，通过设置合理的切片时间，可以大大加速较大视频的转码速度。 -您可以实现如下需求: +FFmpeg 是音视频处理领域的一款强大工具，它可以被用于格式转换、编解码、录制和流处理等多种音视频相关任务。该技术因其高效性和多功能性，在视频处理、直播流媒体、数字媒体播放和编辑等多个领域中得到广泛应用。截至2023年4月，FFmpeg 在 GitHub 上的 star 数接近30,000，这反映了其在开源社区的受欢迎程度和影响力。被Netflix、YouTube、Facebook等多家知名企业采用，这些公司依赖 FFmpeg 处理其庞大的视频数据，以提供流畅的媒体播放体验、视频内容编辑和优化视频传输效率等服务。 -- 一个视频文件可以同时被转码成各种格式以及其他各种自定义处理，比如增加水印处理或者在 after-process 更新信息到数据库等。 + -- 当有多个文件同时上传到 OSS，函数计算会自动伸缩， 并行处理多个文件， 同时每次文件转码成多种格式也是并行。 +## 使用流程 -- 结合 NAS + 视频切片， 可以解决超大视频的转码， 对于每一个视频，先进行切片处理，然后并行转码切片，最后合成，通过设置合理的切片时间，可以大大加速较大视频的转码速度。 + -![image](https://img.alicdn.com/tfs/TB1A.PSzrj1gK0jSZFuXXcrHpXa-570-613.png) +上传一个 mov 格式的视频到对象存储 bucket, OSS 触发器自动触发函数执行，函数调用工作流执行，工作流会同时进行 1 种或者多种格式的转码(由 s.yaml 中的 DST_FORMATS 参数控制)， 本示例默认配置的是同时进行 mp4, flv, avi 格式的转码。 - +基于该工作流，您可以实现如下需求: -## 使用文档 +1 一个视频文件可以同时被转码成各种格式以及其他各种自定义处理，比如增加水印处理或者在 after-process 更新信息到数据库等。 + +2 当有多个文件同时上传到 OSS，函数计算会自动伸缩， 并行处理多个文件， 同时每次文件转码成多种格式也是并行。 + +3 结合 NAS + 视频切片， 可以解决超大视频的转码， 对于每一个视频，先进行切片处理，然后并行转码切片，最后合成，通过设置合理的切片时间，可以大大加速较大视频的转码速度。 + +![image](https://img.alicdn.com/tfs/TB1A.PSzrj1gK0jSZFuXXcrHpXa-570-613.png) - **操作视频教程:** -[![Watch the video](https://img.alicdn.com/imgextra/i2/O1CN01XvnqJu1XLS8SAU7LT_!!6000000002907-2-tps-250-155.png)](http://devsapp.functioncompute.com/video/video-process-flow.mp4) +[![Watch the video](https://img.alicdn.com/imgextra/i2/O1CN01XvnqJu1XLS8SAU7LT_!!6000000002907-2-tps-250-155.png)](http://devsapp.functioncompute.com/video/multimedia-process-flow.mp4) -**P.S.** 当您想要仅在一个简单的函数中直接完成视频处理逻辑时，可以参考[音视频转码 Job](https://github.com/devsapp/start-ffmpeg/tree/master/transcode) +**P.S.** 当您想要仅在一个简单的函数中直接完成视频处理逻辑时，可以参考[音视频转码 Job](https://github.com/devsapp/start-ffmpeg/tree/V3/transcode/src) +## 注意事项 + + + + @@ -135,7 +118,7 @@

-| | | | +| | | | | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | |

微信公众号：`serverless`
|
微信小助手：`xiaojiangwh`
|
钉钉交流群：`33947367`
|

diff --git a/video-process-flow/src/s.yaml b/video-process-flow/src/s.yaml index b5db1fa..2548041 100644 --- a/video-process-flow/src/s.yaml +++ b/video-process-flow/src/s.yaml @@ -1,124 +1,105 @@ -# ------------------------------------ -# 欢迎您使用阿里云函数计算 FC 组件进行项目开发 -# 组件仓库地址：https://github.com/devsapp/fc -# 组件帮助文档：https://www.serverless-devs.com/fc/readme -# Yaml参考文档：https://www.serverless-devs.com/fc/yaml/readme -# 关于： -# - Serverless Devs和FC组件的关系、如何声明/部署多个函数、超过50M的代码包如何部署 -# - 关于.fcignore使用方法、工具中.s目录是做什么、函数进行build操作之后如何处理build的产物 -# 等问题，可以参考文档：https://www.serverless-devs.com/fc/tips -# 关于如何做CICD等问题，可以参考：https://www.serverless-devs.com/serverless-devs/cicd -# 关于如何进行环境划分等问题，可以参考：https://www.serverless-devs.com/serverless-devs/extend -# 更多函数计算案例，可参考：https://github.com/devsapp/awesome/ -# 有问题快来钉钉群问一下吧：33947367 -# ------------------------------------ - -edition: 1.0.0 -name: video-process-flow -# access 是当前应用所需要的密钥信息配置： -# 密钥配置可以参考：https://www.serverless-devs.com/serverless-devs/command/config -# 密钥使用顺序可以参考：https://www.serverless-devs.com/serverless-devs/tool#密钥使用顺序与规范 -access: "{{ access }}" - +edition: 3.0.0 +name: multimedia-process-flow-v3 +access: '{{ access }}' vars: - region: "{{ region }}" - service: - name: "{{ serviceName }}" - description: use fc+fnf+ffmpeg to transcode video in FC - internetAccess: true - role: "{{ serviceRoleArn }}" - nasConfig: auto - # logConfig: auto - flowName: "{{ flowName }}" + region: '{{ region }}' + flowName: '{{ flowName }}' + role: '{{ functionRoleArn }}' -services: - # 函数计算配置 - fc-video-demo-split: - component: devsapp/fc +resources: + split: + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: split - handler: index.handler - timeout: 600 - memorySize: 3072 - runtime: python3 - codeUri: code/split - fc-video-demo-transcode: - component: devsapp/fc + handler: index.handler + timeout: 600 + memorySize: 3072 + runtime: python3 + functionName: ${vars.flowName}_split + role: ${vars.role} + code: code/split + vpcConfig: auto + nasConfig: auto + + + transcode: + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: transcode - handler: index.handler - timeout: 600 - memorySize: 3072 - runtime: python3 - codeUri: code/transcode - fc-video-demo-merge: - component: devsapp/fc + handler: index.handler + timeout: 600 + memorySize: 3072 + runtime: python3 + functionName: ${vars.flowName}_transcode + role: ${vars.role} + code: code/transcode + vpcConfig: ${resources.split.output.vpcConfig} + nasConfig: ${resources.split.output.nasConfig} + + merge: + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: merge - handler: index.handler - timeout: 600 - memorySize: 3072 - runtime: python3 - codeUri: code/merge - fc-video-demo-after-process: - component: devsapp/fc + handler: index.handler + timeout: 600 + memorySize: 3072 + runtime: python3 + functionName: ${vars.flowName}_merge + role: ${vars.role} + code: code/merge + vpcConfig: ${resources.split.output.vpcConfig} + nasConfig: ${resources.split.output.nasConfig} + + after-process: + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: after-process - handler: index.handler - timeout: 120 - memorySize: 512 - runtime: python3 - codeUri: code/after-process - fc-oss-trigger-trigger-fnf: - component: devsapp/fc + handler: index.handler + timeout: 120 + memorySize: 512 + runtime: python3 + functionName: ${vars.flowName}_after-process + role: ${vars.role} + code: code/after-process + vpcConfig: ${resources.split.output.vpcConfig} + nasConfig: ${resources.split.output.nasConfig} + + oss-trigger-workflow: + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: trigger-fnf - handler: index.handler - timeout: 120 - memorySize: 128 - runtime: python3 - codeUri: code/oss-trigger - environmentVariables: - OUTPUT_DST: '{{ outputDir }}' - FLOW_NAME: ${vars.flowName} - SEG_INTERVAL: '{{ segInterval }}' - DST_FORMATS: '{{ dstFormats }}' + handler: index.handler + timeout: 120 + memorySize: 128 + runtime: python3 + environmentVariables: + OUTPUT_DST: '{{ outputDir }}' + FLOW_NAME: ${vars.flowName} + SEG_INTERVAL: '{{ segInterval }}' + DST_FORMATS: '{{ dstFormats }}' + functionName: ${vars.flowName}_oss-trigger-workflow + role: ${vars.role} + code: code/oss-trigger triggers: - - name: oss-t - type: oss - role: '{{ triggerRoleArn }}' - config: + - triggerName: ${vars.flowName} + triggerType: oss + triggerConfig: events: - oss:ObjectCreated:PutObject - oss:ObjectCreated:PostObject - oss:ObjectCreated:CompleteMultipartUpload filter: - Key: - Prefix: '{{ prefix }}' - Suffix: '' - bucketName: '{{ ossBucket }}' + key: + prefix: '{{ prefix }}' + suffix: '' + invocationRole: '{{ triggerRoleArn }}' + sourceArn: acs:oss:${this.props.region}:${config("AccountID")}:{{ ossBucket }} - # fnf 服务配置 - video-demo-flow: - component: devsapp/fnf + multimedia-process-v3-workflow: + component: flow props: name: ${vars.flowName} region: ${vars.region} - description: FnF video processing demo flow - definition: code/flows/video-processing-fc.yml - roleArn: "{{ fnfRoleArn }}" \ No newline at end of file + description: Serverless workflow video process + definition: ${yaml(file('code/workflows/multimedia-process-workflow.yml'))} + roleArn: '{{ fnfRoleArn }}'