让浏览器自己工作：AI 自动化技术落地全攻略【AI 助力全员提效方向】

async function testLogin(page) {  await page.fill('#username', 'testuser');  await page.fill('#password', 'Pass123!');  await page.click('#login-btn');  await expect(page).toHaveURL(/dashboard/);}

async function smartLogin(page, ai) {  const context = {    pageHTML: await page.content(),    task: "完成登录操作",    constraints: "使用有效测试凭证"  };    const plan = await ai.generateActionPlan(context);    for (const action of plan.actions) {    if (action.type === 'fill') {      const element = await ai.locateElement({        page: page,        description: action.field      });      await element.fill(await ai.generateTestData(action.field));    }    // 其他动作类型处理...  }    const result = await ai.verifyOutcome({    page: page,    expected: "成功登录"  });}

npm install @midscene/web puppeteer tsx --save-dev

import puppeteer from "puppeteer";import { PuppeteerAgent } from "@midscene/web/puppeteer";
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));Promise.resolve(  (async () => {    const browser = await puppeteer.launch({      headless: false, // here we use headed mode to help debug    });
    const page = await browser.newPage();    await page.setViewport({      width: 1280,      height: 800,      deviceScaleFactor: 1,    });
    await page.goto("https://www.ebay.com");    await sleep(5000);
    // 👀 初始化 Midscene agent     const agent = new PuppeteerAgent(page);
    // 👀 执行搜索    // 注：尽管这是一个英文页面，你也可以用中文指令控制它    await agent.aiAction('在搜索框输入 "Headphones" ，敲回车');    await sleep(5000);
    // 👀 理解页面，提取数据    const items = await agent.aiQuery(      '{itemTitle: string, price: Number}[], 找到列表里的商品标题和价格',    );    console.log("耳机商品信息", items);
    // 👀 用 AI 断言    await agent.aiAssert("界面左侧有类目筛选功能");
    await browser.close();  })());

npm install @midscene/web playwright @playwright/test tsx --save-dev 

import { chromium } from 'playwright';import { PlaywrightAgent } from '@midscene/web/playwright';import 'dotenv/config'; // read environment variables from .env file
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
Promise.resolve(  (async () => {    const browser = await chromium.launch({      headless: true, // 'true' means we can't see the browser window      args: ['--no-sandbox', '--disable-setuid-sandbox'],    });
    const page = await browser.newPage();    await page.setViewportSize({      width: 1280,      height: 768,    });    await page.goto('https://www.ebay.com');    await sleep(5000); // 👀 init Midscene agent    const agent = new PlaywrightAgent(page);
    // 👀 type keywords, perform a search    await agent.aiAction('type "Headphones" in search box, hit Enter');
    // 👀 wait for the loading    await agent.aiWaitFor('there is at least one headphone item on page');    // or you may use a plain sleep:    // await sleep(5000);
    // 👀 understand the page content, find the items    const items = await agent.aiQuery(      '{itemTitle: string, price: Number}[], find item in list and corresponding price',    );    console.log('headphones in stock', items);
    const isMoreThan1000 = await agent.aiBoolean(      'Is the price of the headphones more than 1000?',    );    console.log('isMoreThan1000', isMoreThan1000);
    const price = await agent.aiNumber(      'What is the price of the first headphone?',    );    console.log('price', price);
    const name = await agent.aiString(      'What is the name of the first headphone?',    );    console.log('name', name);
    const location = await agent.aiLocate(      'What is the location of the first headphone?',    );    console.log('location', location);
    // 👀 assert by AI    await agent.aiAssert('There is a category filter on the left');
    // 👀 click on the first item    await agent.aiTap('the first item in the list');
    await browser.close();  })(),);

npm install @midscene/web tsx --save-dev

import { AgentOverChromeBridge } from "@midscene/web/bridge-mode";
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));Promise.resolve(  (async () => {    const agent = new AgentOverChromeBridge();
    // 这个方法将连接到你的桌面 Chrome 的新标签页    // 记得启动你的 Chrome 插件，并点击 'allow connection' 按钮。否则你会得到一个 timeout 错误    await agent.connectNewTabWithUrl("https://www.bing.com");
    // 这些方法与普通 Midscene agent 相同    await agent.ai('type "AI 101" and hit Enter');    await sleep(3000);
    await agent.aiAssert("there are some search results");    await agent.destroy();  })());

tsx demo-new-tab.ts

MIDSCENE_CACHE=1 playwright test --config=playwright.config.ts

{  "mcpServers": {    "mcp-midscene": {      "command": "npx",      "args": ["-y", "@midscene/mcp"],      "env": {        "MIDSCENE_MODEL_NAME": "REPLACE_WITH_YOUR_MODEL_NAME",        "OPENAI_API_KEY": "REPLACE_WITH_YOUR_OPENAI_API_KEY",        "MCP_SERVER_REQUEST_TIMEOUT": "800000"      }    }  }}

交互方法agent.aiAction() 或 .ai()  # UI 操作步骤agent.aiTap()   #点击某个元素agent.aiHover()  # 鼠标悬停某个元素上agent.aiInput()  # 在某个元素中输入文本agent.aiKeyboardPress()  # 按下键盘上的某个键、agent.aiScroll()  # 滚动页面或某个元素agent.aiRightClick() # 右键点击某个元素数据提取agent.aiAsk()  # 针对当前页面，直接向 AI 模型发起提问，并获得字符串形式的回答agent.aiQuery() # 直接从 UI 提取结构化的数据agent.aiBoolean() # 从 UI 中提取一个布尔值agent.aiNumber()  # 从 UI 中提取一个数字agent.aiString()  # 从 UI 中提取一个字符串断言等agent.aiAssert()  # 进行结果断言
更多参考官网