feat: Visual diff support approve progress (#48994)

* refactor: adjust failed build

* feat: approve check

* chore: checkbox accept

* chore: update check

* chore: clean up

* chore: typo

.github/workflows/visual-regression-diff-approver.yml

@@ -0,0 +1,64 @@
+# Check the report passed or developer confirmed the diff
+
+name: 🤖 Visual Regression Diff Approver
+
+# on comment event
+on:
+  issue_comment:
+    types: [created, edited, deleted]
+
+permissions:
+  contents: read
+
+jobs:
+  rebase:
+    permissions:
+      pull-requests: read
+    name: Check Virtual Regression Approval
+    if: github.event.issue.pull_request != ''
+    runs-on: ubuntu-latest
+    steps:
+    # Check all the comments if contains `VISUAL_DIFF_SUCCESS` or `VISUAL_DIFF_FAILED`
+    # Check if member of repo comment `Pass Visual Diff`
+    # Return type: `success` or `failed` or `waiting`
+    - name: Statistic Visual Diff Approval
+      id: check_approval
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const { data: comments } = await github.rest.issues.listComments({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+          });
+
+          let hasDiffSuccess = false;
+          let hasDiffFailed = false;
+          let hasMemberApprove = false;
+
+          for (const comment of comments) {
+            if (comment.body.includes('VISUAL_DIFF_SUCCESS')) {
+              hasDiffSuccess = true;
+            }
+            if (comment.body.includes('VISUAL_DIFF_FAILED')) {
+              hasDiffFailed = true;
+            }
+            if (comment.body.includes('- [x] Visual diff is acceptable')) {
+              hasMemberApprove = true;
+            }
+          }
+
+          if (hasDiffSuccess || (hasDiffFailed && hasMemberApprove)) {
+            return 'success';
+          } else if (hasDiffFailed) {
+            return 'failed';
+          } else {
+            return 'waiting';
+          }
+
+    # Console need approve message and throw failure if the result is `failed`
+    - name: Fail if Visual Diff Failed
+      if: steps.check_approval.outputs.result == 'failed'
+      run: |
+        echo "Visual Diff Failed, please check the diff and approve it."
+        exit 1

scripts/visual-regression/build.ts

@@ -6,7 +6,6 @@ import os from 'os';
 import path from 'path';
 import { Readable } from 'stream';
 import { finished } from 'stream/promises';
-import simpleGit from 'simple-git';
 import chalk from 'chalk';
 import fse from 'fs-extra';
 import difference from 'lodash/difference';
@@ -14,6 +13,7 @@ import minimist from 'minimist';
 import pixelmatch from 'pixelmatch';
 import { PNG } from 'pngjs';
 import sharp from 'sharp';
+import simpleGit from 'simple-git';
 
 import markdown2Html from './convert';
 
@@ -248,12 +248,13 @@ function generateReport(
   const passed = badCases.length === 0;
 
   const commonHeader = `
+<!-- ${passed ? 'VISUAL_DIFF_SUCCESS' : 'VISUAL_DIFF_FAILED'} -->
+
 ## 👁 Visual Regression Report for PR #${prId} ${passed ? 'Passed ✅' : 'Failed ❌'}
 > **🎯 Target branch:** ${targetBranch} (${targetRef})
   `.trim();
 
   const htmlReportLink = `${publicPath}/report.html`;
-  const addonFullReportDesc = `\n\nCheck <a href="${htmlReportLink}" target="_blank">Full Report</a> for details`;
 
   const fullReport = `> 📖 <a href="${htmlReportLink}" target="_blank">View Full Report ↗︎</a>`;
   if (passed) {
@@ -291,7 +292,19 @@ ${fullReport}
     fullVersionMd += generateLineReport(badCase, publicPath, currentRef, false);
   }
 
-  reportMdStr += addonFullReportDesc;
+  reportMdStr += `\n\nCheck <a href="${htmlReportLink}" target="_blank">Full Report</a> for details`;
+
+  // tips for comment `Pass Visual Diff` will pass the CI
+  if (!passed) {
+    reportMdStr += `
+
+-----
+
+If you think the visual diff is acceptable, please check:
+
+- [ ] Visual diff is acceptable
+`;
+  }
 
   // convert fullVersionMd to html
   return [reportMdStr, markdown2Html(fullVersionMd)];
@@ -472,8 +485,9 @@ async function boot() {
   console.log(chalk.red('⛔️ Failed cases:\n'));
   console.log(prettyList(sortedBadCases.map((i) => `[${i.type}] ${i.filename}`)));
   console.log('\n');
-  // let job failed
-  process.exit(1);
+
+  // let job failed. Skip to let CI/CD to handle it
+  // process.exit(1);
 }
 
 boot();