Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,30 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212455.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212504.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212514.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212605.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212612.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212613.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212622.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212631.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212732.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212738.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212739.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212759.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212809.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212949.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212956.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_212957.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_213032.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_213042.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251103_213052.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251104_155252.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251104_155301.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251104_155302.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251104_155330.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251104_155340.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251106_144427.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251106_144436.png filter=lfs diff=lfs merge=lfs -text
+logs/screenshot_20251106_144437.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/main.yaml

@@ -0,0 +1,49 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: CI Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test-pytest:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run pytest
+      run: |
+        pytest
+

.gitignore

@@ -0,0 +1,177 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Node stuff
+node_modules/

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# How to contribute
+
+We'd love to accept your patches and contributions to this project.
+
+## Before you begin
+
+### Sign our Contributor License Agreement
+
+Contributions to this project must be accompanied by a
+[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).
+You (or your employer) retain the copyright to your contribution; this simply
+gives us permission to use and redistribute your contributions as part of the
+project.
+
+If you or your current employer have already signed the Google CLA (even if it
+was for a different project), you probably don't need to do it again.
+
+Visit <https://cla.developers.google.com/> to see your current agreements or to
+sign a new one.
+
+### Review our community guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
+
+## Contribution process
+
+### Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Google LLC
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,145 @@
 ---
-title: Computer Use Preview
-emoji: 🐨
-colorFrom: gray
-colorTo: gray
+title: computer-use-preview
+app_file: app.py
 sdk: gradio
 sdk_version: 5.49.1
-app_file: app.py
-pinned: false
 ---
+# Computer Use Preview
+
+## Quick Start
+
+This section will guide you through setting up and running the Computer Use Preview model, either the Gemini Developer API or Vertex AI. Follow these steps to get started.
+
+### 1. Installation
+
+**Clone the Repository**
+
+```bash
+git clone https://github.com/google/computer-use-preview.git
+cd computer-use-preview
+```
+
+**Set up Python Virtual Environment and Install Dependencies**
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+**Install Playwright and Browser Dependencies**
+
+```bash
+# Install system dependencies required by Playwright for Chrome
+playwright install-deps chrome
+
+# Install the Chrome browser for Playwright
+playwright install chrome
+```
+
+### 2. Configuration
+You can get started using either the Gemini Developer API or Vertex AI.
+
+#### A. If using the Gemini Developer API:
+
+You need a Gemini API key to use the agent:
+
+```bash
+export GEMINI_API_KEY="YOUR_GEMINI_API_KEY"
+```
+
+Or to add this to your virtual environment:
+
+```bash
+echo 'export GEMINI_API_KEY="YOUR_GEMINI_API_KEY"' >> .venv/bin/activate
+# After editing, you'll need to deactivate and reactivate your virtual
+# environment if it's already active:
+deactivate
+source .venv/bin/activate
+```
+
+Replace `YOUR_GEMINI_API_KEY` with your actual key.
+
+#### B. If using the Vertex AI Client:
+
+You need to explicitly use Vertex AI, then provide project and location to use the agent:
+
+```bash
+export USE_VERTEXAI=true
+export VERTEXAI_PROJECT="YOUR_PROJECT_ID"
+export VERTEXAI_LOCATION="YOUR_LOCATION"
+```
+
+Or to add this to your virtual environment:
+
+```bash
+echo 'export USE_VERTEXAI=true' >> .venv/bin/activate
+echo 'export VERTEXAI_PROJECT="your-project-id"' >> .venv/bin/activate
+echo 'export VERTEXAI_LOCATION="your-location"' >> .venv/bin/activate
+# After editing, you'll need to deactivate and reactivate your virtual
+# environment if it's already active:
+deactivate
+source .venv/bin/activate
+```
+
+Replace `YOUR_PROJECT_ID` and `YOUR_LOCATION` with your actual project and location.
+
+### 3. Running the Tool
+
+The primary way to use the tool is via the `main.py` script.
+
+**General Command Structure:**
+
+```bash
+python main.py --query "Go to Google and type 'Hello World' into the search bar"
+```
+
+**Available Environments:**
+
+You can specify a particular environment with the ```--env <environment>``` flag.  Available options:
+
+- `playwright`: Runs the browser locally using Playwright.
+- `browserbase`: Connects to a Browserbase instance.
+
+**Local Playwright**
+
+Runs the agent using a Chrome browser instance controlled locally by Playwright.
+
+```bash
+python main.py --query="Go to Google and type 'Hello World' into the search bar" --env="playwright"
+```
+
+You can also specify an initial URL for the Playwright environment:
+
+```bash
+python main.py --query="Go to Google and type 'Hello World' into the search bar" --env="playwright" --initial_url="https://www.google.com/search?q=latest+AI+news"
+```
+
+**Browserbase**
+
+Runs the agent using Browserbase as the browser backend. Ensure the proper Browserbase environment variables are set:`BROWSERBASE_API_KEY` and `BROWSERBASE_PROJECT_ID`.
+
+```bash
+python main.py --query="Go to Google and type 'Hello World' into the search bar" --env="browserbase"
+```
+
+## Agent CLI
+
+The `main.py` script is the command-line interface (CLI) for running the browser agent.
+
+### Command-Line Arguments
+
+| Argument | Description | Required | Default | Supported Environment(s) |
+|-|-|-|-|-|
+| `--query` | The natural language query for the browser agent to execute. | Yes | N/A | All |
+| `--env` | The computer use environment to use. Must be one of the following: `playwright`, or `browserbase` | No | N/A | All |
+| `--initial_url` | The initial URL to load when the browser starts. | No | https://www.google.com | All |
+| `--highlight_mouse` | If specified, the agent will attempt to highlight the mouse cursor's position in the screenshots. This is useful for visual debugging. | No | False (not highlighted) | `playwright` |
+
+### Environment Variables
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+| Variable | Description | Required |
+|-|-|-|
+| GEMINI_API_KEY | Your API key for the Gemini model. | Yes |
+| BROWSERBASE_API_KEY | Your API key for Browserbase. | Yes (when using the browserbase environment) |
+| BROWSERBASE_PROJECT_ID | Your Project ID for Browserbase. | Yes (when using the browserbase environment) |

agent.py

@@ -0,0 +1,568 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from typing import Literal, Optional, Union, Any
+from google import genai
+from google.genai import types
+import termcolor
+from google.genai.types import (
+    Part,
+    GenerateContentConfig,
+    Content,
+    Candidate,
+    FunctionResponse,
+    FinishReason,
+)
+import time
+from rich.console import Console
+from rich.table import Table
+
+from computers import EnvState, Computer
+
+MAX_RECENT_TURN_WITH_SCREENSHOTS = 5
+PREDEFINED_COMPUTER_USE_FUNCTIONS = [
+    "open_web_browser",
+    "click_at",
+    "hover_at",
+    "type_text_at",
+    "scroll_document",
+    "scroll_at",
+    "wait_5_seconds",
+    "go_back",
+    "go_forward",
+    "search",
+    "navigate",
+    "key_combination",
+    "drag_and_drop",
+]
+
+
+console = Console()
+
+# Built-in Computer Use tools will return "EnvState".
+# Custom provided functions will return "dict".
+FunctionResponseT = Union[EnvState, dict]
+
+
+def multiply_numbers(x: float, y: float) -> dict:
+    """Multiplies two numbers.
+
+    Args:
+        x (float): The first number.
+        y (float): The second number.
+
+    Returns:
+        dict: The result of multiplication.
+    """
+    return {"result": x * y}
+
+# def multiply_numbers(x: float, y: float) -> dict:
+#     """Multiplies two numbers.
+
+#     Args:
+#         x (float): The first number.
+#         y (float): The second number.
+
+#     Returns:
+#         dict: The result of multiplication.
+#     """
+#     return {"result": x * y}
+
+# 在从types.FunctionDeclaration.from_callable中创建函数声明时，这里的__doc__中所有信息都会被转化为description, 即使其风格符合google style, 但不会进行进一步的解析成参数和返回值的说明
+# 如果参数含义复杂需要在其中再补充一下说明
+def evaluate_answer(query: str, answer:str) -> None:
+    """
+        Evaluates the answer based on the given query.
+    """
+    return None 
+
+class BrowserAgent:
+    def __init__(
+        self,
+        browser_computer: Computer,
+        query: str,
+        system_prompt: str,
+        model_name: str,
+        verbose: bool = True,
+    ):
+        self._browser_computer = browser_computer
+        self._query = query
+        self._model_name = model_name
+        self._verbose = verbose
+        self.final_reasoning = None
+        self._client = genai.Client(
+            api_key=os.environ.get("GEMINI_API_KEY"),
+            vertexai=os.environ.get("USE_VERTEXAI", "0").lower() in ["true", "1"],
+            project=os.environ.get("VERTEXAI_PROJECT"),
+            location=os.environ.get("VERTEXAI_LOCATION"),
+        )
+        self._contents: list[Content] = [
+            Content(
+                role="user",
+                parts=[
+                    Part(text=self._query),
+                ],
+            )
+        ]
+
+        # Exclude any predefined functions here.
+        excluded_predefined_functions = ["scroll_document"] # 先屏蔽使用这个滚动函数
+
+        # Add your own custom functions here.
+        custom_functions = [
+            # For example:
+            types.FunctionDeclaration.from_callable(
+                client=self._client, callable=multiply_numbers
+            )
+        ]
+        
+
+        self._generate_content_config = GenerateContentConfig(
+            temperature=1,
+            top_p=0.95,
+            top_k=40,
+            max_output_tokens=8192,
+            tools=[
+                types.Tool(
+                    computer_use=types.ComputerUse(
+                        environment=types.Environment.ENVIRONMENT_BROWSER,
+                        excluded_predefined_functions=excluded_predefined_functions,
+                    ),
+                ),
+                types.Tool(function_declarations=custom_functions),
+            ],
+            system_instruction=[system_prompt] # 定义system prompt
+            
+        )
+
+    def handle_action(self, action: types.FunctionCall) -> FunctionResponseT:
+        """Handles the action and returns the environment state."""
+        if action.name == "open_web_browser":
+            return self._browser_computer.open_web_browser()
+        elif action.name == "click_at":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.click_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "hover_at":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            return self._browser_computer.hover_at(
+                x=x,
+                y=y,
+            )
+        elif action.name == "type_text_at":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            press_enter = action.args.get("press_enter", False)
+            clear_before_typing = action.args.get("clear_before_typing", True)
+            return self._browser_computer.type_text_at(
+                x=x,
+                y=y,
+                text=action.args["text"],
+                press_enter=press_enter,
+                clear_before_typing=clear_before_typing,
+            )
+        elif action.name == "scroll_document":
+            return self._browser_computer.scroll_document(action.args["direction"])
+        elif action.name == "scroll_at":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            magnitude = action.args.get("magnitude", 800)
+            direction = action.args["direction"]
+
+            if direction in ("up", "down"):
+                magnitude = self.denormalize_y(magnitude)
+            elif direction in ("left", "right"):
+                magnitude = self.denormalize_x(magnitude)
+            else:
+                raise ValueError("Unknown direction: ", direction)
+            return self._browser_computer.scroll_at(
+                x=x, y=y, direction=direction, magnitude=magnitude
+            )
+        elif action.name == "wait_5_seconds":
+            return self._browser_computer.wait_5_seconds()
+        elif action.name == "go_back":
+            return self._browser_computer.go_back()
+        elif action.name == "go_forward":
+            return self._browser_computer.go_forward()
+        elif action.name == "search":
+            return self._browser_computer.search()
+        elif action.name == "navigate":
+            return self._browser_computer.navigate(action.args["url"])
+        elif action.name == "key_combination":
+            return self._browser_computer.key_combination(
+                action.args["keys"].split("+")
+            )
+        elif action.name == "drag_and_drop":
+            x = self.denormalize_x(action.args["x"])
+            y = self.denormalize_y(action.args["y"])
+            destination_x = self.denormalize_x(action.args["destination_x"])
+            destination_y = self.denormalize_y(action.args["destination_y"])
+            return self._browser_computer.drag_and_drop(
+                x=x,
+                y=y,
+                destination_x=destination_x,
+                destination_y=destination_y,
+            )
+        # Handle the custom function declarations here.
+        elif action.name == multiply_numbers.__name__:
+            return multiply_numbers(x=action.args["x"], y=action.args["y"])
+        else:
+            raise ValueError(f"Unsupported function: {action}")
+
+    def get_model_response(
+        self, max_retries=5, base_delay_s=1
+    ) -> types.GenerateContentResponse:
+        for attempt in range(max_retries):
+            try:
+                response = self._client.models.generate_content(
+                    model=self._model_name,
+                    contents=self._contents,
+                    config=self._generate_content_config,
+                )
+                return response  # Return response on success
+            except Exception as e:
+                print(e)
+                if attempt < max_retries - 1:
+                    delay = base_delay_s * (2**attempt)
+                    message = (
+                        f"Generating content failed on attempt {attempt + 1}. "
+                        f"Retrying in {delay} seconds...\n"
+                    )
+                    termcolor.cprint(
+                        message,
+                        color="yellow",
+                    )
+                    time.sleep(delay)
+                else:
+                    termcolor.cprint(
+                        f"Generating content failed after {max_retries} attempts.\n",
+                        color="red",
+                    )
+                    raise
+
+    def get_text(self, candidate: Candidate) -> Optional[str]:
+        """Extracts the text from the candidate."""
+        if not candidate.content or not candidate.content.parts:
+            return None
+        text = []
+        # Gemini 在多模态的场景下，会分段返回内容，例如，除了text和function_call之外，也可能包含其他的部分.
+        # {
+        #   "content": {
+        #     "parts": [
+        #       {"text": "I’ll start by searching Google for relevant documentation."},
+        #       {"function_call": {...}},
+        #       {"text": "Now that I found it, here’s the explanation:"},
+        #       {"text": "In Python, decorators are higher-order functions..."}
+        #     ]
+        #   }
+        # }
+        for part in candidate.content.parts:
+            if part.text:
+                text.append(part.text)
+        return " ".join(text) or None
+
+    def extract_function_calls(self, candidate: Candidate) -> list[types.FunctionCall]:
+        """Extracts the function call from the candidate."""
+        if not candidate.content or not candidate.content.parts:
+            return []
+        ret = []
+        for part in candidate.content.parts:
+            if part.function_call:
+                ret.append(part.function_call)
+        return ret
+
+    # reasoning, screenshot_base64, action, status, url
+    def run_one_iteration_modify(self):
+        # Generate a response from the model.
+        try:
+            response = self.get_model_response()
+        except Exception as e:
+            return "Error occurred", "COMPLETE", []
+
+        if not response.candidates:
+            return "No candidates in response", "COMPLETE", []
+
+        candidate = response.candidates[0]
+        if candidate.content:
+            self._contents.append(candidate.content)
+
+        reasoning = self.get_text(candidate)
+        function_calls = self.extract_function_calls(candidate)
+
+        if not function_calls:
+            self.final_reasoning = reasoning
+            return reasoning, "COMPLETE", []
+
+        # Process each function call and collect results
+        function_responses_list = []
+        status = "CONTINUE"
+        
+        for function_call in function_calls:
+            try:
+                fc_result = self.handle_action(function_call)
+                
+                screenshot_base64 = ""
+                url = ""
+                response = {}
+                if isinstance(fc_result, EnvState): # 浏览器操作事件
+                    screenshot_base64 = fc_result.screenshot
+                    url = fc_result.url
+                    response = {"url": url}
+                    function_response = FunctionResponse(
+                        name=function_call.name,
+                        response={"url": url},
+                        parts=[types.FunctionResponsePart(
+                            inline_data=types.FunctionResponseBlob(
+                                mime_type="image/png", data=screenshot_base64
+                            )
+                        )]
+                    )
+                    self._contents.append(Content(
+                        role="user",
+                        parts=[Part(function_response=function_response)],
+                    ))
+                elif isinstance(fc_result, dict): # 自定义函数
+                    response = fc_result
+                    self._contents.append(Content(
+                        role="user",
+                        parts=[Part(function_response=FunctionResponse(
+                            name=function_call.name,
+                            response=fc_result
+                        ))],
+                    ))
+
+                # Add to function_responses_list
+                function_responses_list.append({
+                    "screenshot": screenshot_base64,
+                    "action": function_call.name,
+                    "response": response,
+                })
+                    
+            except Exception as e:
+                return f"Error handling action {function_call.name}: {str(e)}", "COMPLETE", function_responses_list
+            
+        # only keep screenshots in the few most recent turns, remove the screenshot images from the old turns.
+        turn_with_screenshots_found = 0
+        for content in reversed(self._contents):
+            if content.role == "user" and content.parts:
+                # check if content has screenshot of the predefined computer use functions.
+                has_screenshot = False
+                for part in content.parts:
+                    if (
+                        part.function_response
+                        and part.function_response.parts
+                        and part.function_response.name
+                        in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                    ):
+                        has_screenshot = True
+                        break
+
+                if has_screenshot:
+                    turn_with_screenshots_found += 1
+                    # remove the screenshot image if the number of screenshots exceed the limit.
+                    if turn_with_screenshots_found > MAX_RECENT_TURN_WITH_SCREENSHOTS:
+                        for part in content.parts:
+                            if (
+                                part.function_response
+                                and part.function_response.parts
+                                and part.function_response.name
+                                in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                            ):
+                                part.function_response.parts = None
+
+        return reasoning, status, function_responses_list
+
+    def run_one_iteration(self) -> Literal["COMPLETE", "CONTINUE"]:
+
+        # Generate a response from the model.
+        if self._verbose:
+            with console.status(
+                "Generating response from Gemini Computer Use...", spinner_style=None
+            ):
+                try:
+                    response = self.get_model_response()
+                except Exception as e:
+                    return "COMPLETE"
+        else:
+            try:
+                response = self.get_model_response()
+            except Exception as e:
+                return "COMPLETE"
+
+        if not response.candidates:
+            print("Response has no candidates!")
+            print(response)
+            raise ValueError("Empty response")
+
+        # Extract the text and function call from the response.
+        candidate = response.candidates[0]
+        # Append the model turn to conversation history.
+        if candidate.content:
+            self._contents.append(candidate.content)
+
+        reasoning = self.get_text(candidate)
+        function_calls = self.extract_function_calls(candidate)
+
+        # Retry the request in case of malformed FCs.
+        if (
+            not function_calls
+            and not reasoning
+            and candidate.finish_reason == FinishReason.MALFORMED_FUNCTION_CALL
+        ):
+            return "CONTINUE"
+
+        if not function_calls:
+            print(f"Agent Loop Complete: {reasoning}")
+            self.final_reasoning = reasoning
+            return "COMPLETE"
+
+        function_call_strs = []
+        for function_call in function_calls:
+            # Print the function call and any reasoning.
+            function_call_str = f"Name: {function_call.name}"
+            if function_call.args:
+                function_call_str += f"\nArgs:"
+                for key, value in function_call.args.items():
+                    function_call_str += f"\n  {key}: {value}"
+            function_call_strs.append(function_call_str)
+
+        table = Table(expand=True)
+        table.add_column(
+            "Gemini Computer Use Reasoning", header_style="magenta", ratio=1,
+            no_wrap=False, # 允许换行
+            overflow="fold", # 超出部分折行显示
+        )
+        table.add_column("Function Call(s)", header_style="cyan", ratio=1)
+        table.add_row(reasoning, "\n".join(function_call_strs))
+        if self._verbose:
+            console.print(table)
+            print()
+
+        function_responses = []
+        for function_call in function_calls:
+            extra_fr_fields = {}
+            if function_call.args and (
+                safety := function_call.args.get("safety_decision")
+            ):
+                decision = self._get_safety_confirmation(safety)
+                if decision == "TERMINATE":
+                    print("Terminating agent loop")
+                    return "COMPLETE"
+                # Explicitly mark the safety check as acknowledged.
+                extra_fr_fields["safety_acknowledgement"] = "true"
+            if self._verbose:
+                with console.status(
+                    "Sending command to Computer...", spinner_style=None
+                ):
+                    fc_result = self.handle_action(function_call)
+            else:
+                fc_result = self.handle_action(function_call)
+            if isinstance(fc_result, EnvState):
+                function_responses.append(
+                    FunctionResponse(
+                        name=function_call.name,
+                        response={
+                            "url": fc_result.url,
+                            **extra_fr_fields,
+                        },
+                        parts=[
+                            types.FunctionResponsePart(
+                                inline_data=types.FunctionResponseBlob(
+                                    mime_type="image/png", data=fc_result.screenshot
+                                )
+                            )
+                        ],
+                    )
+                )
+            elif isinstance(fc_result, dict):
+                function_responses.append(
+                    FunctionResponse(name=function_call.name, response=fc_result)
+                )
+
+        self._contents.append(
+            Content(
+                role="user",
+                parts=[Part(function_response=fr) for fr in function_responses],
+            )
+        )
+
+        # only keep screenshots in the few most recent turns, remove the screenshot images from the old turns.
+        turn_with_screenshots_found = 0
+        for content in reversed(self._contents):
+            if content.role == "user" and content.parts:
+                # check if content has screenshot of the predefined computer use functions.
+                has_screenshot = False
+                for part in content.parts:
+                    if (
+                        part.function_response
+                        and part.function_response.parts
+                        and part.function_response.name
+                        in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                    ):
+                        has_screenshot = True
+                        break
+
+                if has_screenshot:
+                    turn_with_screenshots_found += 1
+                    # remove the screenshot image if the number of screenshots exceed the limit.
+                    if turn_with_screenshots_found > MAX_RECENT_TURN_WITH_SCREENSHOTS:
+                        for part in content.parts:
+                            if (
+                                part.function_response
+                                and part.function_response.parts
+                                and part.function_response.name
+                                in PREDEFINED_COMPUTER_USE_FUNCTIONS
+                            ):
+                                part.function_response.parts = None
+
+        return "CONTINUE"
+
+    def _get_safety_confirmation(
+        self, safety: dict[str, Any]
+    ) -> Literal["CONTINUE", "TERMINATE"]:
+        if safety["decision"] != "require_confirmation":
+            raise ValueError(f"Unknown safety decision: safety['decision']")
+        termcolor.cprint(
+            "Safety service requires explicit confirmation!",
+            color="yellow",
+            attrs=["bold"],
+        )
+        print(safety["explanation"])
+        decision = ""
+        while decision.lower() not in ("y", "n", "ye", "yes", "no"):
+            decision = input("Do you wish to proceed? [Yes]/[No]\n")
+        if decision.lower() in ("n", "no"):
+            return "TERMINATE"
+        return "CONTINUE"
+
+    def agent_loop(self):
+        status = "CONTINUE"
+        while status == "CONTINUE":
+            status = self.run_one_iteration()
+    
+    def agent_loop_yield(self):
+        status = "CONTINUE"
+        while status == "CONTINUE":
+            reasoning, status, function_responses_list = self.run_one_iteration_modify()
+            yield reasoning, status, function_responses_list
+
+    def denormalize_x(self, x: int) -> int:
+        return int(x / 1000 * self._browser_computer.screen_size()[0])
+
+    def denormalize_y(self, y: int) -> int:
+        return int(y / 1000 * self._browser_computer.screen_size()[1])

app.py

@@ -0,0 +1,200 @@
+import gradio as gr
+import time
+import base64
+from datetime import datetime
+import argparse
+import os
+
+from agent import BrowserAgent
+from computers import BrowserbaseComputer, PlaywrightComputer
+import base64
+import sys
+
+sys.path.append(os.path.dirname(__file__))
+
+from prompt import prompt_options
+
+PLAYWRIGHT_SCREEN_SIZE = (1440, 900)
+
+os.environ["PLAYWRIGHT_HEADLESS"] = "false"
+
+def run_genflow(query:str, prompt:str = ""):
+    env = PlaywrightComputer(
+        screen_size=PLAYWRIGHT_SCREEN_SIZE,
+        initial_url="https://wenku.baidu.com/ndcore/browse/aiunion?fr=options_AIcard_1&_wkts_=1761290807747&bdQuery=genflow&t=1761290807744&tabType=genflow&aiCreat=genflow",
+        # initial_url="https://www.doubao.com/chat/",
+        highlight_mouse=True, # 如果指定，代理将尝试在屏幕截图中突出显示鼠标光标的位置。这对于可视化调试很有用。
+    )
+    # query="在提示文本为'输入问题，交给GenFlow的'搜索条中输入‘百度’，点击发送按钮",
+    
+    prompt = "GenFlow是一个AI聊天机器人。你需要作为测试员，在它的网页底部搜索栏中输入用户提交的问题, 并且观察其输出结果。" + prompt
+    prompt += """
+GenFlow 返回的内容可能比较长，你可以多次执行'scroll_at'操作来查看网页中它返回的上下文。
+GenFlow 有时会呈现左边主页面，右边预览区域。在这种情况下执行'scroll_at'的时候你要注意鼠标位置。
+通过反复的滚动，确保主页面和预览区域已经滚动到底。预览区域往往很长，所以你需要多次执行'scroll_at'操作，确保2次滚动操作看到的网页完全一样为止, 给出最终客观评价.
+
+GenFlow 一定会给出'输出结果'，请保持足够的耐心！
+**无论输入的语言是什么，你需要输出中文**"""
+    # print(prompt)
+    with env as browser_computer:
+        agent = BrowserAgent(
+            browser_computer=browser_computer,
+            query=query,
+            system_prompt=prompt,
+            model_name='gemini-2.5-computer-use-preview-10-2025',
+        )
+        # reasoning, status, function_responses_list
+        for step in agent.agent_loop_yield():
+            # print(step)
+            # input("☕️")
+            yield step
+
+# === 处理用户查询 ===
+def process_user_query(query, prompt=""):
+    if not query.strip():
+        yield [], "请输入有效的查询", "<p>无执行步骤</p>"
+        return
+
+    steps = []
+
+    for reasoning, status, function_responses_list in run_genflow(query, prompt):
+        # 每个 function_responses_list 是若干个 ["screenshot", "action", "response"]
+        sub_steps = []
+        for item in function_responses_list:
+            if len(item) == 3:
+                screenshot_base64, action, response = item["screenshot"], item["action"], item["response"]
+            else:
+                screenshot_base64, action, response = "", "未知操作", ""
+
+            sub_steps.append({
+                "screenshot": f"data:image/png;base64,{base64.b64encode(screenshot_base64).decode("utf-8")}" if screenshot_base64 else "",
+                "action": action,
+                "response": response,
+            })
+
+        step = {
+            "step": len(steps) + 1,
+            "reasoning": reasoning,
+            "status": status,
+            "functions": sub_steps,  # ✅ 支持多个函数结果
+        }
+        steps.append(step)
+        display_html = update_steps_display(steps)
+        yield steps, f"正在执行第 {len(steps)} 步: {status}", display_html
+        time.sleep(0.3)
+
+    yield steps, f"任务完成！共执行 {len(steps)} 步。", update_steps_display(steps)
+
+# === 更新步骤展示的 HTML ===
+def update_steps_display(steps):
+    """以HTML格式显示每一步的推理、函数动作及截图"""
+    if not steps:
+        return "<p>暂无执行步骤</p>"
+
+    html = "<div style='font-family: Arial, sans-serif;'>"
+    for step in steps:
+        status_color = {
+            "COMPLETE": "green",
+            "CONTINUE": "orange",
+            "FAILURE": "red",
+        }.get(step["status"], "black")
+
+        html += f"""
+        <div style='border:1px solid #ddd; margin:10px 0; padding:15px; border-radius:8px;'>
+            <div style='display:flex; justify-content:space-between; align-items:center;'>
+                <h3 style='margin:0;'>步骤 {step['step']}</h3>
+                <span style='color:{status_color}; font-weight:bold;'>{step['status']}</span>
+            </div>
+            <p><strong>推理:</strong> {step['reasoning']}</p>
+        """
+
+        # ✅ 支持多个函数动作展示
+        for idx, func in enumerate(step["functions"], start=1):
+            # print(func)
+            html += f"""
+            <div style="word-wrap: break-word; word-break: break-all; white-space: pre-wrap; border:1px solid #ddd; border-radius:6px; padding:8px; margin-top:6px;">
+                <p><strong>函数调用 {idx}:</strong></p>
+                <p>动作: {func['action']}</p>
+                <p>返回: {func['response']}</p>
+            """
+            if func["screenshot"]:
+                html += f"<img src='{func['screenshot']}' style='max-width:100%; border:1px solid #ccc; border-radius:6px;'/>"
+            html += "</div>"
+
+        html += "</div>"  # 结束step块
+    html += "</div>"
+    return html
+
+
+
+def resolve_prompt(selected_value):
+    """如果选择的是预定义 key，则返回对应 value，否则直接返回用户输入"""
+    if selected_value in prompt_options:
+        return prompt_options[selected_value]
+    return selected_value or ""
+
+# === 创建 Gradio 界面 ===
+def create_demo_interface():
+    with gr.Blocks(title="CUA 在线试用系统", theme=gr.themes.Soft()) as demo:
+        gr.Markdown("""
+<div style="font-family: 'Microsoft YaHei', sans-serif; font-size: 18px; line-height: 1.6;">
+    <h1 style="color: #2E86AB; font-size: 28px; font-weight: bold; text-align: center;">🧭 CUA 在线试用系统</h1>
+    <p style="color: #555; font-size: 16px; text-align: center;">
+        输入一个任务描述，Agent 将自动进行 genflow 中浏览、截图、分析并执行下一步操作。
+    </p>
+</div>
+""")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                query_input = gr.Textbox(
+                    label="需要在 genflow 中测试的问题",
+                    placeholder="例如：帮我搜索AI新闻 / 购买一台笔记本电脑",
+                    lines=2)
+                
+                prompt_dropdown = gr.Dropdown(
+                    label="选择或在右侧系统提示词中输入评估标准",
+                    choices=list(prompt_options.keys()),
+                    value=None,
+                    allow_custom_value=False,  # ✅ 允许用户手动输入文本
+                )
+                submit_btn = gr.Button("开始执行", variant="primary")
+                
+            with gr.Column(scale=2):
+                status_output = gr.Textbox(label="执行状态", interactive=False)
+                final_prompt = gr.Textbox(label="系统提示词", interactive=True,lines=5)
+                # 按钮或自动触发逻辑
+                prompt_dropdown.change(resolve_prompt, inputs=prompt_dropdown, outputs=final_prompt)
+
+
+            
+
+        # ✅ 把详细执行过程放在 JSON 前面
+        with gr.Accordion("详细执行过程", open=True):
+            steps_display = gr.HTML()
+
+        # 再放执行步骤 JSON
+        steps_output = gr.JSON(label="执行步骤详情（JSON）")
+
+        # 流式输出绑定
+        submit_btn.click(
+            fn=process_user_query,
+            inputs=[query_input, final_prompt],
+            outputs=[steps_output, status_output, steps_display],
+            api_name="run_agent",
+            show_progress=True,
+            queue=True,  # 支持流式输出
+        )
+
+    return demo
+
+
+
+if __name__ == "__main__":
+    demo = create_demo_interface()
+    demo.queue()  # 必须启用 queue 才能支持 yield
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=True,
+    )

computers/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .computer import Computer, EnvState
+from .browserbase.browserbase import BrowserbaseComputer
+from .playwright.playwright import PlaywrightComputer
+
+__all__ = [
+    "Computer",
+    "EnvState",
+    "BrowserbaseComputer",
+    "PlaywrightComputer",
+]

computers/browserbase/browserbase.py

@@ -0,0 +1,80 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import termcolor
+from ..playwright.playwright import PlaywrightComputer
+import browserbase
+from playwright.sync_api import sync_playwright
+
+
+class BrowserbaseComputer(PlaywrightComputer):
+    def __init__(
+        self,
+        screen_size: tuple[int, int],
+        initial_url: str = "https://www.google.com",
+    ):
+        super().__init__(screen_size, initial_url)
+
+    def __enter__(self):
+        print("Creating session...")
+
+        self._playwright = sync_playwright().start()
+        self._browserbase = browserbase.Browserbase(
+            api_key=os.environ["BROWSERBASE_API_KEY"]
+        )
+
+        self._session = self._browserbase.sessions.create(
+            project_id=os.environ["BROWSERBASE_PROJECT_ID"],
+            browser_settings={
+                "fingerprint": {
+                    "screen": {
+                        "maxWidth": 1920,
+                        "maxHeight": 1080,
+                        "minWidth": 1024,
+                        "minHeight": 768,
+                    },
+                },
+                "viewport": {
+                    "width": self._screen_size[0],
+                    "height": self._screen_size[1],
+                },
+            },
+        )
+
+        self._browser = self._playwright.chromium.connect_over_cdp(
+            self._session.connect_url
+        )
+        self._context = self._browser.contexts[0]
+        self._page = self._context.pages[0]
+        self._page.goto(self._initial_url)
+
+        self._context.on("page", self._handle_new_page)
+
+        termcolor.cprint(
+            f"Session started at https://browserbase.com/sessions/{self._session.id}",
+            color="green",
+            attrs=["bold"],
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._page.close()
+
+        if self._context:
+            self._context.close()
+
+        if self._browser:
+            self._browser.close()
+
+        self._playwright.stop()

computers/computer.py

@@ -0,0 +1,125 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import abc
+import pydantic
+from typing import Literal
+
+
+class EnvState(pydantic.BaseModel):
+    # The screenshot in PNG format.
+    screenshot: bytes
+    url: str
+
+
+class Computer(abc.ABC):
+    """Defines an interface for environments."""
+
+    @abc.abstractmethod
+    def screen_size(self) -> tuple[int, int]:
+        """Returns the screen size of the environment."""
+
+    @abc.abstractmethod
+    def open_web_browser(self) -> EnvState:
+        """Opens the web browser."""
+
+    @abc.abstractmethod
+    def click_at(self, x: int, y: int) -> EnvState:
+        """Clicks at a specific x, y  coordinate on the webpage.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def hover_at(self, x: int, y: int) -> EnvState:
+        """Hovers at a specific x, y coordinate on the webpage.
+
+        May be used to explore sub-menus that appear on hover.
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def type_text_at(
+        self,
+        x: int,
+        y: int,
+        text: str,
+        press_enter: bool,
+        clear_before_typing: bool,
+    ) -> EnvState:
+        """Types text at a specific x, y coordinate.
+
+        The system automatically presses ENTER after typing. To disable this, set `press_enter` to False.
+        The system automatically clears any existing content before typing the specified `text`. To disable this, set `clear_before_typing` to False.
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def scroll_document(
+        self, direction: Literal["up", "down", "left", "right"]
+    ) -> EnvState:
+        """Scrolls the entire webpage "up", "down", "left" or "right" based on direction."""
+
+    @abc.abstractmethod
+    def scroll_at(
+        self,
+        x: int,
+        y: int,
+        direction: Literal["up", "down", "left", "right"],
+        magnitude: int,
+    ) -> EnvState:
+        """Scrolls up, down, right, or left at a x, y coordinate by magnitude.
+
+        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def wait_5_seconds(self) -> EnvState:
+        """Waits for 5 seconds to allow unfinished webpage processes to complete."""
+
+    @abc.abstractmethod
+    def go_back(self) -> EnvState:
+        """Navigates back to the previous webpage in the browser history."""
+
+    @abc.abstractmethod
+    def go_forward(self) -> EnvState:
+        """Navigates forward to the next webpage in the browser history."""
+
+    @abc.abstractmethod
+    def search(self) -> EnvState:
+        """Directly jumps to a search engine home page.
+
+        Used when you need to start with a search. For example, this is used when
+        the current website doesn't have the information needed or because a new
+        task is being started.
+        """
+
+    @abc.abstractmethod
+    def navigate(self, url: str) -> EnvState:
+        """Navigates directly to a specified URL."""
+
+    @abc.abstractmethod
+    def key_combination(self, keys: list[str]) -> EnvState:
+        """Presses keyboard keys and combinations, such as "control+c" or "enter"."""
+
+    @abc.abstractmethod
+    def drag_and_drop(
+        self, x: int, y: int, destination_x: int, destination_y: int
+    ) -> EnvState:
+        """Drag and drop an element from a x, y coordinate to a destination destination_y, destination_x coordinate.
+        The 'x', 'y', 'destination_y' and 'destination_x' values are absolute values, scaled to the height and width of the screen.
+        """
+
+    @abc.abstractmethod
+    def current_state(self) -> EnvState:
+        """Returns the current state of the current webpage."""

computers/playwright/playwright.py

@@ -0,0 +1,430 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import termcolor
+import time
+import os
+import sys
+from ..computer import (
+    Computer,
+    EnvState,
+)
+import playwright.sync_api
+from playwright.sync_api import sync_playwright
+from typing import Literal
+import glob
+
+# Define a mapping from the user-friendly key names to Playwright's expected key names.
+# Playwright is generally good with case-insensitivity for these, but it's best to be canonical.
+# See: https://playwright.dev/docs/api/class-keyboard#keyboard-press
+# Keys like 'a', 'b', '1', '$' are passed directly.
+PLAYWRIGHT_KEY_MAP = {
+    "backspace": "Backspace",
+    "tab": "Tab",
+    "return": "Enter",  # Playwright uses 'Enter'
+    "enter": "Enter",
+    "shift": "Shift",
+    "control": "ControlOrMeta",
+    "alt": "Alt",
+    "escape": "Escape",
+    "space": "Space",  # Can also just be " "
+    "pageup": "PageUp",
+    "pagedown": "PageDown",
+    "end": "End",
+    "home": "Home",
+    "left": "ArrowLeft",
+    "up": "ArrowUp",
+    "right": "ArrowRight",
+    "down": "ArrowDown",
+    "insert": "Insert",
+    "delete": "Delete",
+    "semicolon": ";",  # For actual character ';'
+    "equals": "=",  # For actual character '='
+    "multiply": "Multiply",  # NumpadMultiply
+    "add": "Add",  # NumpadAdd
+    "separator": "Separator",  # Numpad specific
+    "subtract": "Subtract",  # NumpadSubtract, or just '-' for character
+    "decimal": "Decimal",  # NumpadDecimal, or just '.' for character
+    "divide": "Divide",  # NumpadDivide, or just '/' for character
+    "f1": "F1",
+    "f2": "F2",
+    "f3": "F3",
+    "f4": "F4",
+    "f5": "F5",
+    "f6": "F6",
+    "f7": "F7",
+    "f8": "F8",
+    "f9": "F9",
+    "f10": "F10",
+    "f11": "F11",
+    "f12": "F12",
+    "command": "Meta",  # 'Meta' is Command on macOS, Windows key on Windows
+}
+
+
+class PlaywrightComputer(Computer):
+    """Connects to a local Playwright instance."""
+
+    def __init__(
+        self,
+        screen_size: tuple[int, int],
+        initial_url: str = "https://www.google.com",
+        search_engine_url: str = "https://www.google.com",
+        highlight_mouse: bool = False,
+    ):
+        self._initial_url = initial_url
+        self._screen_size = screen_size
+        self._search_engine_url = search_engine_url
+        self._highlight_mouse = highlight_mouse
+
+    def _handle_new_page(self, new_page: playwright.sync_api.Page):
+        """The Computer Use model only supports a single tab at the moment.
+
+        Some websites, however, try to open links in a new tab.
+        For those situations, we intercept the page-opening behavior, and instead overwrite the current page.
+        """
+        new_url = new_page.url
+        new_page.close()
+        self._page.goto(new_url) 
+
+    # del
+    def cleanup_old_screenshots(self, logs_dir="logs", max_count=30):
+        """
+        删除旧的截图文件，只保留最新的 max_count 个
+        """
+        # 确保日志目录存在
+        if not os.path.exists(logs_dir):
+            return
+        
+        # 获取所有的png文件
+        pattern = os.path.join(logs_dir, "screenshot_*.png")
+        png_files = glob.glob(pattern)
+        
+        # 如果文件数量超过最大限制，删除旧的
+        if len(png_files) > max_count:
+            # 按修改时间排序，最新的在前面
+            png_files.sort(key=os.path.getmtime, reverse=True)
+            
+            # 要删除的文件（保留最新的 max_count 个）
+            files_to_delete = png_files[max_count:]
+            
+            # 删除旧文件
+            for file_path in files_to_delete:
+                try:
+                    os.remove(file_path)
+                except Exception as e:
+                    raise ValueError(f"Failed to delete screenshots {file_path}: {e}")
+
+    def __enter__(self):
+        print("Creating session...")
+        self._playwright = sync_playwright().start()
+        self._browser = self._playwright.chromium.launch(
+            args=[
+                "--disable-extensions",
+                "--disable-file-system",
+                "--disable-plugins",
+                "--disable-dev-shm-usage",
+                "--disable-background-networking",
+                "--disable-default-apps",
+                "--disable-sync",
+                # No '--no-sandbox' arg means the sandbox is on.
+            ],
+            headless=bool(os.environ.get("PLAYWRIGHT_HEADLESS", False)),
+        )
+        self.cleanup_old_screenshots()
+        os.makedirs("cache/user_data", exist_ok=True) # 定义用户数据存储路径
+        storage_state = "cache/user_data/state.json" if os.path.exists("cache/user_data/state.json") else None
+        self._context = self._browser.new_context(
+            viewport={
+                "width": self._screen_size[0],
+                "height": self._screen_size[1],
+            },
+            storage_state=storage_state
+        )
+            
+        # user_data_dir="cache/user_data", # 保存用户信息，避免重复登录
+        self._page = self._context.new_page()
+        self._page.goto(self._initial_url, wait_until="domcontentloaded")
+        
+        # # 放大110%字体大小，避免看不清
+        # self._page.evaluate("""
+        # (() => {
+        #     const style = document.createElement('style');
+        #     style.innerHTML = `
+        #         * {
+        #             font-size: 100% !important;
+        #         }
+        #     `;
+        #     document.head.appendChild(style);
+        # })();
+        # """)
+
+        # 作为监听器，当self._context打开一个新tab时触发。拦截打开新的tab, 但记录打开的tab需要去到的url, 在当前页面中导航去到新url
+        # 这个CUM在一个时刻只支持一个tab
+        self._context.on("page", self._handle_new_page) 
+
+        # 关于左侧历史消息导航的侧边拦，对齐prompt中对genflow UI的描述，如果不是导航genflow, 可以跳过
+        try:
+            chat_nav = self._page.locator('div[data-v-74198486].chat-nav')
+            is_expand = None
+            if chat_nav.count() > 0:
+                class_name = chat_nav.get_attribute("class")
+                if "show" in class_name:
+                    is_expand = True
+                else:
+                    is_expand = False
+                print(class_name)
+            else:
+                print("未找到左侧历史消息扩展栏元素")
+            if is_expand:
+                expand_icon = self._page.locator('.expand-icon')
+                if expand_icon.count() > 0:
+                    expand_icon.click()
+                    is_expand = False
+                else:
+                    print("未找到左侧历史消息扩展栏点击元素")
+        except Exception as e:
+            raise ValueError("未找到左侧历史消息扩展栏元素") 
+
+        termcolor.cprint(
+            f"Started local playwright.",
+            color="green",
+            attrs=["bold"],
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._context:
+            self._context.storage_state(path="cache/user_data/state.json") # 这里在异常结束的时候是保存不了的
+            self._context.close()
+        try:
+            self._browser.close()
+        except Exception as e:
+            # Browser was already shut down because of SIGINT or such.
+            if "Browser.close: Connection closed while reading from the driver" in str(
+                e
+            ):
+                pass
+            else:
+                raise
+
+        self._playwright.stop()
+
+    def open_web_browser(self) -> EnvState:
+        return self.current_state()
+
+    def click_at(self, x: int, y: int):
+        self.highlight_mouse(x, y)
+        self._page.mouse.click(x, y)
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def hover_at(self, x: int, y: int):
+        self.highlight_mouse(x, y)
+        self._page.mouse.move(x, y)
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def type_text_at(
+        self,
+        x: int,
+        y: int,
+        text: str,
+        press_enter: bool = False,
+        clear_before_typing: bool = True,
+    ) -> EnvState:
+        self.highlight_mouse(x, y)
+        self._page.mouse.click(x, y)
+        self._page.wait_for_load_state()
+
+        if clear_before_typing:
+            if sys.platform == "darwin":
+                self.key_combination(["Command", "A"])
+            else:
+                self.key_combination(["Control", "A"])
+            self.key_combination(["Delete"])
+
+        self._page.keyboard.type(text)
+        self._page.wait_for_load_state()
+
+        if press_enter:
+            self.key_combination(["Enter"])
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def _horizontal_document_scroll(
+        self, direction: Literal["left", "right"]
+    ) -> EnvState:
+        # Scroll by 50% of the viewport size.
+        horizontal_scroll_amount = self.screen_size()[0] // 2
+        if direction == "left":
+            sign = "-"
+        else:
+            sign = ""
+        scroll_argument = f"{sign}{horizontal_scroll_amount}"
+        # Scroll using JS.
+        self._page.evaluate(f"window.scrollBy({scroll_argument}, 0); ")
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def scroll_document(
+        self, direction: Literal["up", "down", "left", "right"]
+    ) -> EnvState:
+        if direction == "down":
+            return self.key_combination(["PageDown"])
+        elif direction == "up":
+            return self.key_combination(["PageUp"])
+        elif direction in ("left", "right"):
+            return self._horizontal_document_scroll(direction)
+        else:
+            raise ValueError("Unsupported direction: ", direction)
+
+    def scroll_at(
+        self,
+        x: int,
+        y: int,
+        direction: Literal["up", "down", "left", "right"],
+        magnitude: int = 800,
+    ) -> EnvState:
+        self.highlight_mouse(x, y)
+
+        self._page.mouse.move(x, y)
+        self._page.wait_for_load_state()
+
+        dx = 0
+        dy = 0
+        if direction == "up":
+            dy = -magnitude
+        elif direction == "down":
+            dy = magnitude
+        elif direction == "left":
+            dx = -magnitude
+        elif direction == "right":
+            dx = magnitude
+        else:
+            raise ValueError("Unsupported direction: ", direction)
+
+        self._page.mouse.wheel(dx, dy)
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def wait_5_seconds(self) -> EnvState:
+        # del
+        time.sleep(20)
+        return self.current_state()
+
+    def go_back(self) -> EnvState:
+        self._page.go_back()
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def go_forward(self) -> EnvState:
+        self._page.go_forward()
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def search(self) -> EnvState:
+        return self.navigate(self._search_engine_url)
+
+    def navigate(self, url: str) -> EnvState:
+        normalized_url = url
+        if not normalized_url.startswith(("http://", "https://")):
+            normalized_url = "https://" + normalized_url
+        self._page.goto(normalized_url)
+        self._page.wait_for_load_state()
+        return self.current_state()
+
+    def key_combination(self, keys: list[str]) -> EnvState:
+        # Normalize all keys to the Playwright compatible version.
+        keys = [PLAYWRIGHT_KEY_MAP.get(k.lower(), k) for k in keys]
+
+        for key in keys[:-1]:
+            self._page.keyboard.down(key)
+
+        self._page.keyboard.press(keys[-1])
+
+        for key in reversed(keys[:-1]):
+            self._page.keyboard.up(key)
+
+        return self.current_state()
+
+    def drag_and_drop(
+        self, x: int, y: int, destination_x: int, destination_y: int
+    ) -> EnvState:
+        self.highlight_mouse(x, y)
+        self._page.mouse.move(x, y)
+        self._page.wait_for_load_state()
+        self._page.mouse.down()
+        self._page.wait_for_load_state()
+
+        self.highlight_mouse(destination_x, destination_y)
+        self._page.mouse.move(destination_x, destination_y)
+        self._page.wait_for_load_state()
+        self._page.mouse.up()
+        return self.current_state()
+
+    def current_state(self) -> EnvState:
+        self._page.wait_for_load_state()
+        # Even if Playwright reports the page as loaded, it may not be so.
+        # Add a manual sleep to make sure the page has finished rendering.
+        time.sleep(0.5)
+        screenshot_bytes = self._page.screenshot(type="png", full_page=False)
+
+        # del 保存截图
+        from datetime import datetime
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"logs/screenshot_{timestamp}.png"
+        with open(filename, "wb") as f:
+            f.write(screenshot_bytes)
+
+        return EnvState(screenshot=screenshot_bytes, url=self._page.url)
+
+    def screen_size(self) -> tuple[int, int]:
+        viewport_size = self._page.viewport_size
+        # If available, try to take the local playwright viewport size.
+        if viewport_size:
+            return viewport_size["width"], viewport_size["height"]
+        # If unavailable, fall back to the original provided size.
+        return self._screen_size
+
+    def highlight_mouse(self, x: int, y: int):
+        if not self._highlight_mouse:
+            return
+        self._page.evaluate(
+            f"""
+        () => {{
+            const element_id = "playwright-feedback-circle";
+            const div = document.createElement('div');
+            div.id = element_id;
+            div.style.pointerEvents = 'none';
+            div.style.border = '4px solid red';
+            div.style.borderRadius = '50%';
+            div.style.width = '20px';
+            div.style.height = '20px';
+            div.style.position = 'fixed';
+            div.style.zIndex = '9999';
+            document.body.appendChild(div);
+
+            div.hidden = false;
+            div.style.left = {x} - 10 + 'px';
+            div.style.top = {y} - 10 + 'px';
+
+            setTimeout(() => {{
+                div.hidden = true;
+            }}, 2000);
+        }}
+    """
+        )
+        # Wait a bit for the user to see the cursor.
+        time.sleep(1)

genflow.py

@@ -0,0 +1,178 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import os
+
+from agent import BrowserAgent
+from computers import BrowserbaseComputer, PlaywrightComputer
+
+
+PLAYWRIGHT_SCREEN_SIZE = (1440, 900)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run the browser agent with a query.")
+    parser.add_argument(
+        "--query",
+        type=str,
+        required=True,
+        help="The query for the browser agent to execute.",
+    )
+
+    parser.add_argument(
+        "--env",
+        type=str,
+        choices=("playwright", "browserbase"),
+        default="playwright",
+        help="The computer use environment to use.",
+    )
+    parser.add_argument(
+        "--initial_url",
+        type=str,
+        default="https://www.google.com",
+        help="The inital URL loaded for the computer.",
+    )
+    parser.add_argument(
+        "--highlight_mouse",
+        action="store_true",
+        default=False,
+        help="If possible, highlight the location of the mouse.",
+    )
+    parser.add_argument(
+        "--model",
+        default='gemini-2.5-computer-use-preview-10-2025',
+        help="Set which main model to use.",
+    )
+    args = parser.parse_args()
+
+
+    if args.env == "playwright":
+        env = PlaywrightComputer(
+            screen_size=PLAYWRIGHT_SCREEN_SIZE,
+            initial_url=args.initial_url,
+            highlight_mouse=args.highlight_mouse,
+        )
+    elif args.env == "browserbase":
+        env = BrowserbaseComputer(
+            screen_size=PLAYWRIGHT_SCREEN_SIZE,
+            initial_url=args.initial_url
+        )
+    else:
+        raise ValueError("Unknown environment: ", args.env)
+
+    with env as browser_computer:
+        agent = BrowserAgent(
+            browser_computer=browser_computer,
+            query=args.query,
+            model_name=args.model,
+        )
+        agent.agent_loop()
+    return 0
+
+
+def run_genflow():
+    env = PlaywrightComputer(
+        screen_size=PLAYWRIGHT_SCREEN_SIZE,
+        initial_url="https://wenku.baidu.com/ndcore/browse/aiunion?fr=options_AIcard_1&_wkts_=1761290807747&bdQuery=genflow&t=1761290807744&tabType=genflow&aiCreat=genflow",
+        # initial_url="https://www.doubao.com/chat/",
+        highlight_mouse=True, # 如果指定，代理将尝试在屏幕截图中突出显示鼠标光标的位置。这对于可视化调试很有用。
+    )
+    # query="在提示文本为'输入问题，交给GenFlow的'搜索条中输入‘百度’，点击发送按钮",
+    search_query = [
+        "《劳动法》中关于加班与调休的相关规定解读",
+        "不到喀什游不算到新疆，不到古城游不算到喀什，最后一天来一场深度的早到晚古城游！",
+        "为何说地球正处在5大灾难性生态临界点？",
+        "为什么要进行展示设计案例分析",
+        "镜子为什么不能正对床？",
+        "鹅绒羽绒服比鸭绒羽绒服好在哪？",
+        "如何从根本解决宿舍问题，改善职工住宿条件？",
+        "独生子女为什么不能全部继承父母遗产？",
+        "写一篇忧郁文艺小诗 一节六句话",
+        "9.3阅兵有哪些国家参加",
+        "今年戛纳电影节有哪些国家的影片参展",
+        "最近的世界移动通信大会有哪些国家的企业参会",
+        "今年日本东京国际动漫节吸引了哪些国家的动漫团队参与",
+        "今年年大阪·关西世界园艺博览会有哪些国家设立展馆",
+        "最近德国法兰克福国际车展有哪些国家的汽车品牌亮相",
+        "第九届世界无人机大会有哪些国家的企业和团队参加",
+        "最近亚洲物流展CeMAT有哪些国家的物流企业参展",
+        "最近世界教育创新峰会有哪些国家的教育机构代表出席",
+        "2025年意大利美食节有哪些国家带来特色美食展示"
+    ]
+    query = "帮我生成一份关于新能源领域的投资报告"
+    citerion = """
+核心评价维度：
+
+1. 内容与逻辑 (灵魂)
+   - 清晰的投资论点: 报告是否有明确、可验证的“买入/卖出”观点及目标价？观点是否新颖，提供了市场尚未察觉的洞察？
+   - 严谨的逻辑链条: 从行业分析到公司研究，再到财务估值，所有论据是否像积木一样牢固支撑核心论点？推理过程是否严密，无逻辑跳跃？
+   - 研究的深度与广度: 是否覆盖了行业（格局、驱动）、公司���模式、护城河）、财务（三张表、比率）和估值（假设合理性）？
+
+2. 数据与事实 (基石)
+   - 准确性与时效性: 数据是否准确？来源（年报、权威统计）是否可靠？信息是否最新？
+   - 相关性与解读: 数据是否与论点强相关？是否对数据背后的业务含义进行了深入解读，而非简单罗列？
+
+3. 结构与表达 (骨架)
+   - 结构清晰: 是否有清晰的摘要、目录和逻辑流程？（例如：摘要->行业->公司->财务->估值->风险->结论）
+   - 语言专业: 语言是否精炼、客观、准确？专业术语使用是否恰当？
+   - 可视化: 图表是否清晰、直观？排版是否专业，重点突出？
+
+4. 客观性与风险意识 (灵魂试金石)
+   - 客观中立: 是否同时呈现了正反论据？是否存在明显偏见？
+   - 全面的风险分析: 是否识别并深入分析了关键的下行风险（行业、公司、宏观、估值）？是否讨论了风险对投资论点的影响？"""
+    
+    query = "查找5个简历模板链接"
+    citerion = "通过访问链接确保每一条都是有效的, 并且资源链接的数量是否满足5条。有时候链接通过页面卡片提供，你需要通过点击卡片进行链接跳转。"
+    prompt = f"""
+GenFlow是一个AI聊天机器人。对于给定的问题，他会执行'任务规划'，'任务执行'，'输出结果'三个步骤。
+在GenFlow执行每个步骤时，'任务规划'，'任务执行'，'输出结果'会分别呈现黑色或灰色。黑色代表该步骤完成，灰色代表该步骤未完成。
+你需要作为测试员，在它的网页底部搜索栏中输入用户提交的问题，在GenFlow执行完三个任务后，客观地判断其最终是否按照给定标准:{citerion} 回答了问题。
+
+GenFlow返回的内容可能比较长，你可以多次执行'scroll_at'操作来查看网页中它返回的上下文。
+GenFlow有时会呈现左边主页面，右边预览区域。在这种情况下执行'scroll_at'的时候你要注意鼠标位置。
+通过反复的滚动，确保主页面和预览区域已经滚动到底。预览区域往往很长，所以你需要多次执行'scroll_at'操作，确保2次滚动操作看到的网页完全一样为止, 给出最终客观评价.
+
+GenFlow一定会给出'输出结果'，请保持足够的耐心！
+Regardless of the input language, you need to output in Chinese.
+"""
+    
+    citerion = """提交任务"1+1，2+3和4+5",判断结果分别是不是2，5，9"""
+    query = f"""比较genflow和doubao哪个在数学计算上的能力更好。标准: {citerion}"""
+    prompt = f"""
+GenFlow和DouBao是2个AI聊天机器人。
+你需要作为测试员，根据用户需求在它们的网页端搜索栏中输入问题，评测它们的能力
+初始状态你处于DouBao的首页，在测试完毕后，你需要切换到GenFlow的首页进行测试。然后给出最终结论
+DouBao的url是 <url>https://www.doubao.com/chat/</url>
+GenFlow的url是 <url>https://wenku.baidu.com/ndcore/browse/aiunion?fr=options_AIcard_1&_wkts_=1761290807747&bdQuery=genflow&t=1761290807744&tabType=genflow&aiCreat=genflow"</url>
+
+AI返回的内容可能比较长，你可以多次执行'scroll_at'操作来查看网页中它们返回的上下文。
+你需要多次执行'scroll_at'操作，确保2次滚动操作看到的网页完全一样为止, 给出最终客观评价.
+
+Regardless of the input language, you need to output in Chinese.
+ """
+    
+    with env as browser_computer:
+        agent = BrowserAgent(
+            browser_computer=browser_computer,
+            query=query,
+            system_prompt=prompt,
+            model_name='gemini-2.5-computer-use-preview-10-2025',
+        )
+        agent.agent_loop()
+    # pass
+
+
+
+if __name__ == "__main__":
+    run_genflow()