{"id":1277,"date":"2025-12-04T08:49:13","date_gmt":"2025-12-04T00:49:13","guid":{"rendered":"http:\/\/www.preluna.xyz\/?p=1277"},"modified":"2025-12-04T09:12:07","modified_gmt":"2025-12-04T01:12:07","slug":"agentserveragent_computer_control","status":"publish","type":"post","link":"http:\/\/www.preluna.xyz\/index.php\/2025\/12\/04\/agentserveragent_computer_control\/preluna\/technology\/career-skills\/open-source-analysis\/","title":{"rendered":"agentserver\\agent_computer_control"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\">computer_control_agent<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>\"\"\"\n\u7535\u8111\u63a7\u5236\u4e3bAgent - \u534f\u8c03\u5404\u4e2a\u7ec4\u4ef6\uff0c\u63d0\u4f9b\u7edf\u4e00\u7684\u7535\u8111\u63a7\u5236\u63a5\u53e3\n\"\"\"\n\nimport asyncio\nimport logging\nfrom typing import Dict, Any, Optional\nimport json #\u7528\u4e8e\u628a\u6570\u636e\u8f6c\u6362\u6210JSON\u683c\u5f0f\uff08\u7f51\u7edc\u4f20\u8f93\u5e38\u7528\uff09\n\nfrom .computer_use_adapter import ComputerUseAdapter\nfrom .visual_analyzer import VisualAnalyzer\nfrom .action_executor import ActionExecutor, ActionResult\n'''\ncomputer_use_adapter.py\uff08\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\uff09\naction_executor.py\uff08\u52a8\u4f5c\u6267\u884c\u5668\uff09\nvisual_analyzer.py\uff08\u89c6\u89c9\u5206\u6790\u5668 - \u6211\u4eec\u8fd8\u6ca1\u770b\uff09\n'''\n\n# \u914d\u7f6e\u65e5\u5fd7\nlogger = logging.getLogger(__name__)\n\nclass ComputerControlAgent:\n    \"\"\"\u7535\u8111\u63a7\u5236\u4e3bAgent\uff0c\u8d1f\u8d23\u534f\u8c03\u5404\u4e2a\u7ec4\u4ef6\"\"\"\n    \n    def __init__(self):\n        \"\"\"\u521d\u59cb\u5316\u7535\u8111\u63a7\u5236Agent\"\"\"\n        self.adapter = ComputerUseAdapter()   # \u63a7\u5236\u4e13\u5bb6,adapter\uff1a\u8d1f\u8d23\u5177\u4f53\u7684\u9f20\u6807\u952e\u76d8\u63a7\u5236\n        self.analyzer = VisualAnalyzer()      # \u89c6\u89c9\u4e13\u5bb6,analyzer\uff1a\u8d1f\u8d23\u5206\u6790\u5c4f\u5e55\u5185\u5bb9\n        self.executor = ActionExecutor(       # \u6267\u884c\u4e13\u5bb6,executor\uff1a\u8d1f\u8d23\u534f\u8c03\u6267\u884c\u52a8\u4f5c\n            computer_adapter=self.adapter,\n            visual_analyzer=self.analyzer\n        )\n        '''\n        \u6267\u884c\u5668 (executor)\n           \u2193\n           \u251c\u2500\u2192 \u9002\u914d\u5668 (adapter)\uff1a\u63a7\u5236\u786c\u4ef6\n           \u2514\u2500\u2192 \u5206\u6790\u5668 (analyzer)\uff1a\u5206\u6790\u5c4f\u5e55\n        '''\n        \n        logger.info(\"\u7535\u8111\u63a7\u5236Agent\u521d\u59cb\u5316\u5b8c\u6210\")\n    \n    async def handle_handoff(self, task: dict) -> str:\n        \"\"\"\u5904\u7406\u7535\u8111\u63a7\u5236\u4efb\u52a1\"\"\"\n        try:\n            logger.info(f\"\u6536\u5230\u7535\u8111\u63a7\u5236\u4efb\u52a1: {task}\")\n            \n            # \u89e3\u6790\u4efb\u52a1\u53c2\u6570\n            action = task.get(\"action\", \"\")\n            target = task.get(\"target\", \"\")\n            parameters = task.get(\"parameters\", {})\n            \n            # \u6839\u636e\u52a8\u4f5c\u7c7b\u578b\u5904\u7406\n            if action == \"click\":\n                return await self._handle_click(target, parameters)\n            elif action == \"click_ai\":\n                return await self._handle_click_ai(target, parameters)\n            elif action == \"type\":\n                return await self._handle_type(target, parameters)\n            elif action == \"screenshot\":\n                return await self._handle_screenshot(target, parameters)\n            elif action == \"find_element\":\n                return await self._handle_find_element(target, parameters)\n            elif action == \"locate_ai\":\n                return await self._handle_locate_ai(target, parameters)\n            elif action == \"automate_task\":\n                return await self._handle_automate_task(target, parameters)\n            elif action == \"coordinate_info\":\n                return await self._handle_coordinate_info(target, parameters)\n            else:\n                return await self._handle_generic_task(action, target, parameters)\n            '''\n            \u8fd9\u662f\u6574\u4e2a\u7cfb\u7edf\u7684\"\u5927\u95e8\"\uff1a\n              1.\u63a5\u6536\u4e00\u4e2a\u4efb\u52a1\uff08\u5b57\u5178\u683c\u5f0f\uff09\n              2.\u89e3\u6790\u51fa\u4e09\u4e2a\u90e8\u5206\uff1a\n                 action\uff1a\u8981\u505a\u4ec0\u4e48\uff08\u70b9\u51fb\u3001\u6253\u5b57\u7b49\uff09\n                 target\uff1a\u76ee\u6807\u662f\u4ec0\u4e48\n                 parameters\uff1a\u989d\u5916\u53c2\u6570\n              3.\u6839\u636e\u52a8\u4f5c\u7c7b\u578b\uff0c\u5206\u53d1\u7ed9\u4e13\u95e8\u7684\u5904\u7406\u51fd\u6570\n            '''\n                \n        except Exception as e:\n            logger.error(f\"\u5904\u7406\u7535\u8111\u63a7\u5236\u4efb\u52a1\u5931\u8d25: {e}\")\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u4efb\u52a1\u5904\u7406\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    # \u70b9\u51fb\u5904\u7406\u51fd\u6570\uff08\u4e24\u79cd\u70b9\u51fb\u65b9\u5f0f\uff09\n    async def _handle_click(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u70b9\u51fb\u4efb\u52a1\uff0c\u652f\u6301AI\u5b9a\u4f4d\u548c\u591a\u79cd\u5750\u6807\u683c\u5f0f\"\"\"\n        try:\n            # \u68c0\u67e5\u662f\u5426\u4f7f\u7528AI\u5b9a\u4f4d\n            use_ai_location = parameters.get(\"use_ai\", False)\n            ai_description = parameters.get(\"ai_description\", target)\n            \n            #\u7279\u70b9\uff1a\u544a\u8bc9AI\"\u70b9\u51fb\u4fdd\u5b58\u6309\u94ae\"\uff0cAI\u81ea\u5df1\u627e\u6309\u94ae\u4f4d\u7f6e\n            if use_ai_location and ai_description:\n                # \u4f7f\u7528AI\u5b9a\u4f4d\u8fdb\u884c\u70b9\u51fb\n                success = await self.adapter.click_with_ai_location(ai_description, parameters.get(\"button\", \"left\"))\n                \n                if success:\n                    return json.dumps({\n                        \"success\": True,\n                        \"message\": f\"AI\u5b9a\u4f4d\u70b9\u51fb\u6210\u529f: {ai_description}\",\n                        \"data\": {\"method\": \"ai_location\", \"target\": ai_description}\n                    }, ensure_ascii=False)\n                else:\n                    return json.dumps({\n                        \"success\": False,\n                        \"error\": \"AI\u5b9a\u4f4d\u5931\u8d25\",\n                        \"message\": f\"AI\u5b9a\u4f4d\u70b9\u51fb\u5931\u8d25: {ai_description}\"\n                    }, ensure_ascii=False)\n            else:\n                # \u4f7f\u7528\u4f20\u7edf\u65b9\u5f0f\u6267\u884c\u70b9\u51fb,\u7279\u70b9\uff1a\u6307\u5b9a\u5177\u4f53\u5750\u6807\u4f4d\u7f6e\u8fdb\u884c\u70b9\u51fb\n                action = {\n                    \"action\": \"click\",\n                    \"target\": target,\n                    \"parameters\": parameters\n                }\n                \n                # \u6267\u884c\u70b9\u51fb\n                result = await self.executor.execute_action(action)\n                \n                if result.success:\n                    return json.dumps({\n                        \"success\": True,\n                        \"message\": f\"\u6210\u529f\u70b9\u51fb: {target}\",\n                        \"data\": result.data\n                    }, ensure_ascii=False)\n                else:\n                    return json.dumps({\n                        \"success\": False,             # \u662f\u5426\u6210\u529f\n                        \"error\": result.error,        # \u7ed3\u679c\u4fe1\u606f\n                        \"message\": result.message     # \u989d\u5916\u6570\u636e\n                    }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u70b9\u51fb\u64cd\u4f5c\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_type(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u8f93\u5165\u4efb\u52a1\"\"\"\n        try:\n            # \u6784\u5efa\u8f93\u5165\u52a8\u4f5c\n            action = {\n                \"action\": \"type\",\n                \"target\": target,\n                \"parameters\": parameters\n            }\n            \n            # \u6267\u884c\u8f93\u5165\n            result = await self.executor.execute_action(action)\n            \n            if result.success:\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"\u6210\u529f\u8f93\u5165: {target}\",\n                    \"data\": result.data\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": result.error,\n                    \"message\": result.message\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u8f93\u5165\u64cd\u4f5c\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_screenshot(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u622a\u56fe\u4efb\u52a1\"\"\"\n        try:\n            # \u6784\u5efa\u622a\u56fe\u52a8\u4f5c\n            action = {\n                \"action\": \"screenshot\",\n                \"target\": target,\n                \"parameters\": parameters\n            }\n            \n            # \u6267\u884c\u622a\u56fe\n            result = await self.executor.execute_action(action)\n            \n            if result.success:\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"\u622a\u56fe\u6210\u529f: {target}\",\n                    \"data\": result.data\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": result.error,\n                    \"message\": result.message\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u622a\u56fe\u64cd\u4f5c\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_find_element(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u5143\u7d20\u67e5\u627e\u4efb\u52a1\"\"\"\n        try:\n            # \u6784\u5efa\u67e5\u627e\u52a8\u4f5c\n            action = {\n                \"action\": \"find_element\",\n                \"target\": target,\n                \"parameters\": parameters\n            }\n            \n            # \u6267\u884c\u67e5\u627e\n            result = await self.executor.execute_action(action)\n            \n            if result.success:\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"\u627e\u5230\u5143\u7d20: {target}\",\n                    \"data\": result.data\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": result.error,\n                    \"message\": result.message\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u5143\u7d20\u67e5\u627e\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_automate_task(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u81ea\u52a8\u5316\u4efb\u52a1\"\"\"\n        try:\n            logger.info(f\"\u5f00\u59cb\u81ea\u52a8\u5316\u4efb\u52a1: {target}\")\n            # \u76f4\u63a5\u8c03\u7528\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u6267\u884c\u81ea\u7136\u8bed\u8a00\u6307\u4ee4\n            exec_result = await self.adapter.run_instruction(target)\n            success = bool(exec_result.get(\"success\")) if isinstance(exec_result, dict) else False\n            return json.dumps({\n                \"success\": success,\n                \"message\": \"\u4efb\u52a1\u6267\u884c\u5b8c\u6210\" if success else exec_result.get(\"error\", \"\u4efb\u52a1\u6267\u884c\u5931\u8d25\"),\n                \"data\": exec_result\n            }, ensure_ascii=False)\n            \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u81ea\u52a8\u5316\u4efb\u52a1\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_generic_task(self, action: str, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u901a\u7528\u4efb\u52a1\"\"\"\n        try:\n            # \u6784\u5efa\u901a\u7528\u52a8\u4f5c\n            action_dict = {\n                \"action\": action,\n                \"target\": target,\n                \"parameters\": parameters\n            }\n            \n            # \u6267\u884c\u52a8\u4f5c\n            result = await self.executor.execute_action(action_dict)\n            \n            if result.success:\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"\u4efb\u52a1\u6267\u884c\u6210\u529f: {action} - {target}\",\n                    \"data\": result.data\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": result.error,\n                    \"message\": result.message\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u901a\u7528\u4efb\u52a1\u6267\u884c\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    def get_capabilities(self) -> Dict&#91;str, Any]:\n        \"\"\"\u83b7\u53d6\u7535\u8111\u63a7\u5236\u80fd\u529b\"\"\"\n        return {\n            \"enabled\": True,\n            \"adapter\": self.adapter.is_available(),\n            \"analyzer\": self.analyzer.is_available(),\n            \"planner\": self.planner.is_available(),\n            \"executor\": self.executor.is_available(),\n            \"capabilities\": &#91;\n                \"\u9f20\u6807\u70b9\u51fb\",\n                \"\u952e\u76d8\u8f93\u5165\",\n                \"\u5c4f\u5e55\u622a\u56fe\",\n                \"\u5143\u7d20\u67e5\u627e\",\n                \"\u4efb\u52a1\u81ea\u52a8\u5316\",\n                \"\u89c6\u89c9\u5206\u6790\"\n            ]\n        }\n    \n    async def _handle_click_ai(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406AI\u5b9a\u4f4d\u70b9\u51fb\u4efb\u52a1\"\"\"\n        try:\n            # \u4f7f\u7528AI\u5b9a\u4f4d\u8fdb\u884c\u70b9\u51fb\n            success = await self.adapter.click_with_ai_location(target, parameters.get(\"button\", \"left\"))\n            \n            if success:\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"AI\u5b9a\u4f4d\u70b9\u51fb\u6210\u529f: {target}\",\n                    \"data\": {\"method\": \"ai_location\", \"target\": target}\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": \"AI\u5b9a\u4f4d\u5931\u8d25\",\n                    \"message\": f\"AI\u5b9a\u4f4d\u70b9\u51fb\u5931\u8d25: {target}\"\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"AI\u5b9a\u4f4d\u70b9\u51fb\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_locate_ai(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406AI\u5b9a\u4f4d\u4efb\u52a1\"\"\"\n        try:\n            # \u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\n            screenshot = await self.adapter.take_screenshot()\n            if not screenshot:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": \"\u622a\u56fe\u5931\u8d25\",\n                    \"message\": \"\u65e0\u6cd5\u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\"\n                }, ensure_ascii=False)\n            \n            # \u4f7f\u7528AI\u5b9a\u4f4d\u5143\u7d20\n            location = await self.analyzer.locate_element_with_ai(\n                target, \n                screenshot, \n                self.adapter.screen_width, \n                self.adapter.screen_height\n            )\n            \n            if location:\n                x, y = location\n                return json.dumps({\n                    \"success\": True,\n                    \"message\": f\"AI\u5b9a\u4f4d\u6210\u529f: {target}\",\n                    \"data\": {\n                        \"target\": target,\n                        \"coordinates\": {\"x\": x, \"y\": y},\n                        \"method\": \"ai_location\"\n                    }\n                }, ensure_ascii=False)\n            else:\n                return json.dumps({\n                    \"success\": False,\n                    \"error\": \"AI\u5b9a\u4f4d\u5931\u8d25\",\n                    \"message\": f\"\u65e0\u6cd5\u5b9a\u4f4d\u5143\u7d20: {target}\"\n                }, ensure_ascii=False)\n                \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"AI\u5b9a\u4f4d\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    async def _handle_coordinate_info(self, target: str, parameters: Dict&#91;str, Any]) -> str:\n        \"\"\"\u5904\u7406\u5750\u6807\u7cfb\u7edf\u4fe1\u606f\u67e5\u8be2\"\"\"\n        try:\n            # \u83b7\u53d6\u5750\u6807\u7cfb\u7edf\u4fe1\u606f\n            coordinate_info = self.adapter.get_coordinate_info()\n            \n            return json.dumps({\n                \"success\": True,\n                \"message\": \"\u5750\u6807\u7cfb\u7edf\u4fe1\u606f\u83b7\u53d6\u6210\u529f\",\n                \"data\": coordinate_info\n            }, ensure_ascii=False)\n            \n        except Exception as e:\n            return json.dumps({\n                \"success\": False,\n                \"error\": str(e),\n                \"message\": f\"\u83b7\u53d6\u5750\u6807\u7cfb\u7edf\u4fe1\u606f\u5931\u8d25: {str(e)}\"\n            }, ensure_ascii=False)\n    \n    def get_status(self) -> Dict&#91;str, Any]:\n        \"\"\"\u83b7\u53d6\u7535\u8111\u63a7\u5236\u72b6\u6001\"\"\"\n        return {\n            \"agent_name\": \"ComputerControlAgent\",\n            \"version\": \"2.0.0\",  # \u5347\u7ea7\u7248\u672c\u53f7\n            \"status\": \"running\",\n            \"capabilities\": self.get_capabilities(),\n            \"components\": {\n                \"adapter\": \"ComputerUseAdapter\",\n                \"analyzer\": \"VisualAnalyzer\", \n                \"planner\": \"TaskPlanner\",\n                \"executor\": \"ActionExecutor\"\n            },\n            \"upgrades\": &#91;\n                \"AI\u5750\u6807\u5b9a\u4f4d\",\n                \"\u5750\u6807\u6807\u51c6\u5316\u7cfb\u7edf\",\n                \"\u591a\u683c\u5f0f\u5750\u6807\u652f\u6301\",\n                \"\u667a\u80fd\u5143\u7d20\u5b9a\u4f4d\"\n            ]\n        }\n\n# ==========\u8fd9\u4e2a\u6587\u4ef6\u521b\u9020\u4e86\u4ec0\u4e48\uff1f==========\n'''\n \u65b0\u7684\u7c7b\uff1aComputerControlAgent\uff08\u603b\u6307\u6325\uff09\n9\u4e2a\u5904\u7406\u51fd\u6570\uff1a\nhandle_handoff\uff08\u4e3b\u5165\u53e3\uff09\n_handle_click\uff08\u70b9\u51fb\uff09\n_handle_click_ai\uff08AI\u70b9\u51fb\uff09\n_handle_type\uff08\u6253\u5b57\uff09\n_handle_screenshot\uff08\u622a\u56fe\uff09\n_handle_find_element\uff08\u67e5\u627e\u5143\u7d20\uff09\n_handle_automate_task\uff08\u81ea\u52a8\u5316\u4efb\u52a1\uff09\n_handle_locate_ai\uff08AI\u5b9a\u4f4d\uff09\n_handle_coordinate_info\uff08\u5750\u6807\u4fe1\u606f\uff09\n_handle_generic_task\uff08\u901a\u7528\u4efb\u52a1\uff09\n2\u4e2a\u72b6\u6001\u51fd\u6570\uff1a\nget_capabilities\uff08\u80fd\u529b\u67e5\u8be2\uff09\nget_status\uff08\u72b6\u6001\u67e5\u8be2\uff09\n\u5b83\u8c03\u7528\u4e86\u54ea\u4e9b\u5916\u90e8\u51fd\u6570\uff1f\nadapter.click_with_ai_location()\uff1aAI\u70b9\u51fb\nadapter.take_screenshot()\uff1a\u622a\u56fe\nadapter.run_instruction()\uff1a\u6267\u884c\u81ea\u7136\u8bed\u8a00\u6307\u4ee4\nadapter.get_coordinate_info()\uff1a\u83b7\u53d6\u5750\u6807\u4fe1\u606f\nanalyzer.locate_element_with_ai()\uff1aAI\u5b9a\u4f4d\u5143\u7d20\nexecutor.execute_action()\uff1a\u6267\u884c\u52a8\u4f5c\n\u5b83\u505a\u4e86\u54ea\u4e9b\u914d\u7f6e\uff1f\n\u65e5\u5fd7\u914d\u7f6e\uff1a\u8bb0\u5f55\u8fd0\u884c\u8fc7\u7a0b\n\u56e2\u961f\u914d\u7f6e\uff1a\u7ec4\u5efa\u4e86\u4e09\u4e2a\u4e13\u5bb6\n\u8fd4\u56de\u683c\u5f0f\u914d\u7f6e\uff1a\u7edf\u4e00\u4f7f\u7528JSON\u683c\u5f0f\n'''\n#\u8fd9\u4e2a\u6587\u4ef6\u662f\u6574\u4e2a\u7cfb\u7edf\u7684\u603b\u5165\u53e3<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">action_executor.py<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>\"\"\"\n\u52a8\u4f5c\u6267\u884c\u5668 - \u6267\u884c\u5177\u4f53\u7684\u9f20\u6807\u952e\u76d8\u64cd\u4f5c\n\u63d0\u4f9b\u5b89\u5168\u3001\u53ef\u9760\u7684\u52a8\u4f5c\u6267\u884c\u529f\u80fd\n\"\"\"\n\nimport asyncio           # \u5f02\u6b65\u5de5\u5177\uff1a\u8ba9\u591a\u4e2a\u4efb\u52a1\u53ef\u4ee5\u540c\u65f6\u8fdb\u884c\nimport logging           # \u65e5\u5fd7\u5de5\u5177\uff1a\u8bb0\u5f55\u7a0b\u5e8f\u8fd0\u884c\u8fc7\u7a0b\nfrom typing import Dict, Any, Optional, Tuple, List  # \u7c7b\u578b\u63d0\u793a\uff1a\u8ba9\u4ee3\u7801\u66f4\u6e05\u6670\nfrom dataclasses import dataclass        # \u6570\u636e\u7c7b\u5de5\u5177\uff1a\u7b80\u5316\u7c7b\u7684\u521b\u5efa\nfrom enum import Enum                    # \u679a\u4e3e\u5de5\u5177\uff1a\u5b9a\u4e49\u56fa\u5b9a\u9009\u9879\n\n# \u914d\u7f6e\u65e5\u5fd7\nlogger = logging.getLogger(__name__)\n\nclass ActionType(Enum):\n    \"\"\"\u52a8\u4f5c\u7c7b\u578b\u679a\u4e3e\"\"\"\n    CLICK = \"click\"   # \u70b9\u51fb\n    TYPE = \"type\"      # \u6253\u5b57\n    SCREENSHOT = \"screenshot\"   # \u622a\u56fe\n    SCROLL = \"scroll\"      # \u6eda\u52a8\n    DRAG = \"drag\"          # \u62d6\u62fd\n    WAIT = \"wait\"           # \u7b49\u5f85\n    FIND_ELEMENT = \"find_element\"    # \u67e5\u627e\u5143\u7d20\n    ANALYZE = \"analyze\"         # \u5206\u6790\n\n@dataclass\nclass ActionResult:\n    \"\"\"\u6570\u636e\u7c7b-\u52a8\u4f5c\u6267\u884c\u7ed3\u679c\"\"\"\n    success: bool         # \u662f\u5426\u6210\u529f\n    message: str          # \u7ed3\u679c\u4fe1\u606f\n    data: Optional&#91;Dict&#91;str, Any]] = None  # \u989d\u5916\u6570\u636e\n    error: Optional&#91;str] = None    # \u9519\u8bef\u4fe1\u606f\n\nclass ActionExecutor:\n    \"\"\"\u52a8\u4f5c\u6267\u884c\u5668\uff0c\u6267\u884c\u5177\u4f53\u7684\u9f20\u6807\u952e\u76d8\u64cd\u4f5c\"\"\"\n    \n    def __init__(self, computer_adapter=None, visual_analyzer=None):\n        \"\"\"\u521d\u59cb\u5316\u52a8\u4f5c\u6267\u884c\u5668\"\"\"\n        self.computer_adapter = computer_adapter   # \u63a7\u5236\u7535\u8111\u7684\u63a5\u53e3\n        self.visual_analyzer = visual_analyzer     # \u5206\u6790\u5c4f\u5e55\u7684\u5de5\u5177\n        self.execution_history = &#91;]                # \u6267\u884c\u5386\u53f2\u8bb0\u5f55\n        self.safety_mode = True                    # \u5b89\u5168\u6a21\u5f0f\u5f00\u5173\n        self.max_retry_count = 3                   # \u6700\u591a\u91cd\u8bd53\u6b21\n    \n    async def execute_action(self, action: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u5177\u4f53\u52a8\u4f5c\"\"\"  \n        #\u8fd9\u4e2a\u51fd\u6570\u662f\u201c\u603b\u6307\u6325\u201d\uff0c\u5b83\uff1a\u63a5\u6536\u4e00\u4e2a\u52a8\u4f5c\u6307\u4ee4\uff08\u5b57\u5178\uff09\uff0c\u68c0\u67e5\u662f\u4ec0\u4e48\u7c7b\u578b\u7684\u52a8\u4f5c\uff0c\u68c0\u67e5\u662f\u5426\u5b89\u5168\uff0c\u6839\u636e\u52a8\u4f5c\u7c7b\u578b\u8c03\u7528\u4e0d\u540c\u7684\u6267\u884c\u51fd\u6570\uff0c\u8bb0\u5f55\u6267\u884c\u7ed3\u679c\n        try:\n            action_type = action.get(\"action\", \"\").lower()\n            target = action.get(\"target\", \"\")\n            parameters = action.get(\"parameters\", {})\n            \n            logger.info(f\"\u6267\u884c\u52a8\u4f5c: {action_type}, \u76ee\u6807: {target}\")\n            \n            # \u5b89\u5168\u68c0\u67e5\n            if self.safety_mode and not self._is_safe_action(action_type, target):  # \u5982\u679c\u5f00\u4e86\u5b89\u5168\u6a21\u5f0f\u4e14\u52a8\u4f5c\u4e0d\u5b89\u5168\uff0c\u5c31\u505c\u6b62\u6267\u884c\n                return ActionResult(\n                    success=False,\n                    message=\"\u52a8\u4f5c\u88ab\u5b89\u5168\u6a21\u5f0f\u963b\u6b62\",\n                    error=\"\u4e0d\u5b89\u5168\u52a8\u4f5c\"\n                )\n            \n            # \u6839\u636e\u52a8\u4f5c\u7c7b\u578b\u6267\u884c\n            if action_type == ActionType.CLICK.value:\n                return await self._execute_click(target, parameters)\n            elif action_type == ActionType.TYPE.value:\n                return await self._execute_type(target, parameters)\n            elif action_type == ActionType.SCREENSHOT.value:\n                return await self._execute_screenshot(target, parameters)\n            elif action_type == ActionType.SCROLL.value:\n                return await self._execute_scroll(target, parameters)\n            elif action_type == ActionType.DRAG.value:\n                return await self._execute_drag(target, parameters)\n            elif action_type == ActionType.WAIT.value:\n                return await self._execute_wait(target, parameters)\n            elif action_type == ActionType.FIND_ELEMENT.value:\n                return await self._execute_find_element(target, parameters)\n            elif action_type == ActionType.ANALYZE.value:\n                return await self._execute_analyze(target, parameters)\n            else:\n                return ActionResult(\n                    success=False,\n                    message=f\"\u672a\u77e5\u52a8\u4f5c\u7c7b\u578b: {action_type}\",\n                    error=\"\u672a\u77e5\u52a8\u4f5c\u7c7b\u578b\"\n                )\n                \n        except Exception as e:\n            logger.error(f\"\u52a8\u4f5c\u6267\u884c\u5931\u8d25: {e}\")\n            return ActionResult(\n                success=False,\n                message=f\"\u52a8\u4f5c\u6267\u884c\u5f02\u5e38: {str(e)}\",\n                error=str(e)\n            )\n        finally:\n            # \u8bb0\u5f55\u6267\u884c\u5386\u53f2\n            self.execution_history.append({\n                \"action\": action,\n                \"timestamp\": asyncio.get_event_loop().time()\n            })\n    \n    async def _execute_click(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u70b9\u51fb\u52a8\u4f5c\uff0c\u652f\u6301\u591a\u79cd\u5750\u6807\u683c\u5f0f\u548cAI\u5b9a\u4f4d\"\"\"\n        try:     #\u68c0\u67e5\u662f\u5426\u6709\u7535\u8111\u63a7\u5236\u5de5\u5177\n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            # \u83b7\u53d6\u5750\u6807\uff0c\u652f\u6301\u591a\u79cd\u683c\u5f0f\n            x = parameters.get(\"x\")\n            y = parameters.get(\"y\")\n            button = parameters.get(\"button\", \"left\")\n            \n            # \u5904\u7406\u5750\u6807\u683c\u5f0f\n            if x is not None and y is not None:    \n                # \u76f4\u63a5\u5750\u6807\n                x, y = self._parse_coordinates(x, y)\n            elif isinstance(target, str) and (target.startswith(('\u70b9\u51fb', 'click', 'Click')) or \n                                           any(keyword in target.lower() for keyword in &#91;'\u6309\u94ae', 'button', '\u56fe\u6807', 'icon'])):\n                # \u5c1d\u8bd5AI\u5b9a\u4f4d\uff0c\u5982\u679c\u6ca1\u6709\u5750\u6807\uff0c\u5c31\u7528AI\u6765\u5bfb\u627e\n                if self.visual_analyzer:\n                    screenshot = await self.computer_adapter.take_screenshot()\n                    location = await self.visual_analyzer.locate_element(target, screenshot)\n                    if location:\n                        x, y = location\n                    else:\n                        return ActionResult(\n                            success=False,\n                            message=f\"AI\u5b9a\u4f4d\u5931\u8d25: {target}\",\n                            error=\"AI\u5b9a\u4f4d\u5931\u8d25\"\n                        )\n                else:\n                    return ActionResult(\n                        success=False,\n                        message=\"\u9700\u8981\u5750\u6807\u6216\u89c6\u89c9\u5206\u6790\u5668\",\n                        error=\"\u7f3a\u5c11\u5fc5\u8981\u53c2\u6570\"\n                    )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7f3a\u5c11\u5750\u6807\u53c2\u6570\",\n                    error=\"\u7f3a\u5c11\u5750\u6807\u53c2\u6570\"\n                )\n            \n            # \u6267\u884c\u70b9\u51fb\n            success = await self.computer_adapter.click(x, y, button)\n            \n            if success:\n                return ActionResult(\n                    success=True,\n                    message=f\"\u6210\u529f\u70b9\u51fb: ({x}, {y})\",\n                    data={\"x\": x, \"y\": y, \"button\": button, \"target\": target}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u70b9\u51fb\u64cd\u4f5c\u5931\u8d25\",\n                    error=\"\u70b9\u51fb\u5931\u8d25\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u70b9\u51fb\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    def _parse_coordinates(self, x, y) -> Tuple&#91;int, int]:\n        \"\"\"\u89e3\u6790\u5750\u6807\uff0c\u652f\u6301\u591a\u79cd\u683c\u5f0f\"\"\"\n        try:\n            # \u5904\u7406\u5b57\u7b26\u4e32\u5750\u6807\n            if isinstance(x, str):\n                x = float(x)\n            if isinstance(y, str):\n                y = float(y)\n            \n            # \u5904\u7406\u5143\u7ec4\/\u5217\u8868\u5750\u6807\n            if isinstance(x, (tuple, list)) and len(x) == 2:\n                x, y = x&#91;0], x&#91;1]\n            \n            # \u8f6c\u6362\u4e3a\u6574\u6570\n            x = int(round(float(x)))\n            y = int(round(float(y)))\n            \n            return x, y\n            \n        except Exception as e:\n            logger.error(f\"\u5750\u6807\u89e3\u6790\u5931\u8d25: {e}\")\n            return 0, 0\n    \n    async def _execute_type(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u8f93\u5165\u52a8\u4f5c\"\"\"\n        try:\n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            text = parameters.get(\"text\", target)\n            interval = parameters.get(\"interval\", 0.1)\n            \n            # \u6267\u884c\u8f93\u5165\n            success = await self.computer_adapter.type_text(text, interval)\n            \n            if success:\n                return ActionResult(\n                    success=True,\n                    message=f\"\u6210\u529f\u8f93\u5165\u6587\u672c: {text}\",\n                    data={\"text\": text, \"interval\": interval}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u6587\u672c\u8f93\u5165\u5931\u8d25\",\n                    error=\"\u8f93\u5165\u5931\u8d25\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u8f93\u5165\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    async def _execute_screenshot(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u622a\u56fe\u52a8\u4f5c\"\"\"\n        try:\n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            # \u6267\u884c\u622a\u56fe\n            screenshot = await self.computer_adapter.take_screenshot()\n            \n            if screenshot:\n                return ActionResult(\n                    success=True,\n                    message=\"\u622a\u56fe\u6210\u529f\",\n                    data={\"screenshot_size\": len(screenshot)}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u622a\u56fe\u5931\u8d25\",\n                    error=\"\u622a\u56fe\u5931\u8d25\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u622a\u56fe\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    async def _execute_scroll(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u6eda\u52a8\u52a8\u4f5c\"\"\"\n        try:\n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            x = parameters.get(\"x\", 0)\n            y = parameters.get(\"y\", 0)\n            clicks = parameters.get(\"clicks\", 3)\n            \n            # \u6267\u884c\u6eda\u52a8\n            success = await self.computer_adapter.scroll(x, y, clicks)\n            \n            if success:\n                return ActionResult(\n                    success=True,\n                    message=f\"\u6210\u529f\u6eda\u52a8: ({x}, {y}), \u6eda\u52a8\u91cf: {clicks}\",\n                    data={\"x\": x, \"y\": y, \"clicks\": clicks}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u6eda\u52a8\u64cd\u4f5c\u5931\u8d25\",\n                    error=\"\u6eda\u52a8\u5931\u8d25\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u6eda\u52a8\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    async def _execute_drag(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u62d6\u62fd\u52a8\u4f5c\"\"\"\n        try:\n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            start_x = parameters.get(\"start_x\", 0)\n            start_y = parameters.get(\"start_y\", 0)\n            end_x = parameters.get(\"end_x\", 100)\n            end_y = parameters.get(\"end_y\", 100)\n            duration = parameters.get(\"duration\", 1.0)\n            \n            # \u6267\u884c\u62d6\u62fd\n            success = await self.computer_adapter.drag_to(start_x, start_y, end_x, end_y, duration)\n            \n            if success:\n                return ActionResult(\n                    success=True,\n                    message=f\"\u6210\u529f\u62d6\u62fd: ({start_x}, {start_y}) -> ({end_x}, {end_y})\",\n                    data={\"start\": (start_x, start_y), \"end\": (end_x, end_y), \"duration\": duration}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u62d6\u62fd\u64cd\u4f5c\u5931\u8d25\",\n                    error=\"\u62d6\u62fd\u5931\u8d25\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u62d6\u62fd\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    async def _execute_wait(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u7b49\u5f85\u52a8\u4f5c\"\"\"\n        try:\n            duration = parameters.get(\"duration\", 1.0)\n            \n            # \u6267\u884c\u7b49\u5f85\n            await asyncio.sleep(duration)\n            \n            return ActionResult(\n                success=True,\n                message=f\"\u7b49\u5f85\u5b8c\u6210: {duration}\u79d2\",\n                data={\"duration\": duration}\n            )\n            \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u7b49\u5f85\u6267\u884c\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    async def _execute_find_element(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u6267\u884c\u5143\u7d20\u67e5\u627e\u52a8\u4f5c\"\"\"\n        try:\n            if not self.visual_analyzer:\n                return ActionResult(\n                    success=False,\n                    message=\"\u89c6\u89c9\u5206\u6790\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u5206\u6790\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            # \u5148\u622a\u56fe\n            screenshot = await self.computer_adapter.take_screenshot()\n            if not screenshot:\n                return ActionResult(\n                    success=False,\n                    message=\"\u622a\u56fe\u5931\u8d25\",\n                    error=\"\u622a\u56fe\u5931\u8d25\"\n                )\n            \n            # \u67e5\u627e\u5143\u7d20\n            location = await self.visual_analyzer.locate_element(target, screenshot)\n            \n            if location:\n                return ActionResult(\n                    success=True,\n                    message=f\"\u627e\u5230\u5143\u7d20: {target} at {location}\",\n                    data={\"location\": location, \"target\": target}\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=f\"\u672a\u627e\u5230\u5143\u7d20: {target}\",\n                    error=\"\u5143\u7d20\u672a\u627e\u5230\"\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u5143\u7d20\u67e5\u627e\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    #\u8f85\u52a9\u51fd\u6570\n    async def _execute_analyze(self, target: str, parameters: Dict&#91;str, Any]) -> ActionResult:\n        \"\"\"\u5750\u6807\u89e3\u6790\u51fd\u6570\u2014\u2014\u6267\u884c\u5206\u6790\u52a8\u4f5c\"\"\"\n        try:\n            if not self.visual_analyzer:\n                return ActionResult(\n                    success=False,\n                    message=\"\u89c6\u89c9\u5206\u6790\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u5206\u6790\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            if not self.computer_adapter:\n                return ActionResult(\n                    success=False,\n                    message=\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\",\n                    error=\"\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n                )\n            \n            # \u5148\u622a\u56fe\n            screenshot = await self.computer_adapter.take_screenshot()\n            if not screenshot:\n                return ActionResult(\n                    success=False,\n                    message=\"\u622a\u56fe\u5931\u8d25\",\n                    error=\"\u622a\u56fe\u5931\u8d25\"\n                )\n            \n            # \u5206\u6790\u5c4f\u5e55\n            analysis = await self.visual_analyzer.analyze_screen(screenshot)\n            \n            if analysis.get(\"success\"):\n                return ActionResult(\n                    success=True,\n                    message=\"\u5c4f\u5e55\u5206\u6790\u5b8c\u6210\",\n                    data=analysis\n                )\n            else:\n                return ActionResult(\n                    success=False,\n                    message=\"\u5c4f\u5e55\u5206\u6790\u5931\u8d25\",\n                    error=analysis.get(\"error\", \"\u5206\u6790\u5931\u8d25\")\n                )\n                \n        except Exception as e:\n            return ActionResult(\n                success=False,\n                message=f\"\u5c4f\u5e55\u5206\u6790\u5931\u8d25: {str(e)}\",\n                error=str(e)\n            )\n    \n    def _is_safe_action(self, action_type: str, target: str) -> bool:\n        \"\"\"\u68c0\u67e5\u52a8\u4f5c\u662f\u5426\u5b89\u5168\"\"\"\n        # \u5371\u9669\u52a8\u4f5c\u5217\u8868\n        dangerous_actions = &#91;\"delete\", \"remove\", \"uninstall\", \"format\", \"shutdown\", \"restart\"]\n        dangerous_targets = &#91;\"\u7cfb\u7edf\", \"system\", \"\u6ce8\u518c\u8868\", \"registry\", \"\u7ba1\u7406\u5458\", \"admin\"]\n        \n        # \u68c0\u67e5\u52a8\u4f5c\u7c7b\u578b\n        if any(danger in action_type.lower() for danger in dangerous_actions):\n            return False\n        \n        # \u68c0\u67e5\u76ee\u6807\n        if any(danger in target.lower() for danger in dangerous_targets):\n            return False\n        \n        return True\n    \n    async def execute_step_sequence(self, steps: list) -> List&#91;ActionResult]:\n        \"\"\"\u6267\u884c\u6b65\u9aa4\u5e8f\u5217\"\"\"\n        results = &#91;]\n        \n        for i, step in enumerate(steps):  #\u904d\u5386\u6bcf\u4e2a\u6b65\u9aa4\n            logger.info(f\"\u6267\u884c\u6b65\u9aa4 {i+1}\/{len(steps)}: {step.get('action', 'unknown')}\")\n            \n            # \u68c0\u67e5\u4f9d\u8d56\uff0c\u68c0\u67e5\u8fd9\u4e2a\u6b65\u9aa4\u6709\u6ca1\u6709\u4f9d\u8d56\u524d\u9762\u7684\u6b65\u9aa4\n            if not self._check_dependencies(step, results):\n                results.append(ActionResult(\n                    success=False,\n                    message=f\"\u6b65\u9aa4 {i+1} \u4f9d\u8d56\u68c0\u67e5\u5931\u8d25\",\n                    error=\"\u4f9d\u8d56\u68c0\u67e5\u5931\u8d25\"\n                ))\n                continue\n            \n            # \u6267\u884c\u6b65\u9aa4\n            result = await self.execute_action(step)\n            results.append(result)\n            \n            # \u5982\u679c\u6b65\u9aa4\u5931\u8d25\uff0c\u53ef\u4ee5\u9009\u62e9\u7ee7\u7eed\u6216\u505c\u6b62\n            if not result.success:\n                logger.warning(f\"\u6b65\u9aa4 {i+1} \u6267\u884c\u5931\u8d25: {result.message}\")\n                # \u8fd9\u91cc\u53ef\u4ee5\u9009\u62e9\u7ee7\u7eed\u6267\u884c\u6216\u505c\u6b62\n                # break  # \u505c\u6b62\u6267\u884c\n                continue  # \u7ee7\u7eed\u6267\u884c\n        \n        return results\n    \n    def _check_dependencies(self, step: dict, previous_results: List&#91;ActionResult]) -> bool:\n        \"\"\"\u68c0\u67e5\u6b65\u9aa4\u4f9d\u8d56\"\"\"\n        dependencies = step.get(\"dependencies\", &#91;])\n        \n        for dep in dependencies:\n            # \u67e5\u627e\u4f9d\u8d56\u7684\u6b65\u9aa4\u7ed3\u679c\n            dep_result = None\n            for result in previous_results:\n                if hasattr(result, 'step_id') and result.step_id == dep:\n                    dep_result = result\n                    break\n            \n            if not dep_result or not dep_result.success:\n                return False\n        \n        return True\n    \n    def is_available(self) -> Dict&#91;str, Any]:\n        \"\"\"\u68c0\u67e5\u52a8\u4f5c\u6267\u884c\u529f\u80fd\u662f\u5426\u53ef\u7528\"\"\"\n        return {\n            \"enabled\": True,\n            \"ready\": self.computer_adapter is not None,\n            \"safety_mode\": self.safety_mode,\n            \"max_retry_count\": self.max_retry_count,\n            \"execution_history_count\": len(self.execution_history)\n        }\n\n# ========\u8fd9\u4e2a->\u662f\u4ec0\u4e48\u4e1c\u897f\uff1f=========\n#\u7bad\u5934 -> \u5728Python\u4e2d\u53eb\u505a\u7c7b\u578b\u63d0\u793a\uff08Type Hints\uff09\uff0c\u4e5f\u53eb\u7c7b\u578b\u6ce8\u89e3\u3002\n#def is_available(self) -> Dict&#91;str, Any]:\n#\u8fd9\u4e2a\u7bad\u5934 -> \u7684\u610f\u601d\u662f\uff1a\"\u8fd9\u4e2a\u51fd\u6570\u4f1a\u8fd4\u56de\u4e00\u4e2a\u2026\u2026\"\uff0c\u628a\u5b83\u8bfb\u6210\uff1a\"\u8fd9\u4e2a\u51fd\u6570\u4f1a\u8fd4\u56de\u4e00\u4e2a Dict&#91;str, Any] \u7c7b\u578b\u7684\u4e1c\u897f\"\n#\u8be6\u7ec6\u5206\u89e3\uff1a\n'''\n1. \u7bad\u5934\u5de6\u8fb9\uff1a\u51fd\u6570\u5b9a\u4e49\npython\ndef is_available(self)\n\u8fd9\u90e8\u5206\u6211\u4eec\u5f88\u719f\u6089\uff0c\u5c31\u662f\u5b9a\u4e49\u4e00\u4e2a\u51fd\u6570\uff0c\u51fd\u6570\u540d\u53eb is_available\n\n2. \u7bad\u5934 ->\uff1a\u8868\u793a\"\u8fd4\u56de\u4ec0\u4e48\u7c7b\u578b\"\n\u53ef\u4ee5\u628a\u5b83\u60f3\u8c61\u6210\u4e00\u4e2a\u65b9\u5411\u6307\u793a\u7bad\u5934\uff0c\u6307\u5411\u51fd\u6570\u8fd4\u56de\u7684\u4e1c\u897f\n\n3. \u7bad\u5934\u53f3\u8fb9\uff1a\u8fd4\u56de\u503c\u7684\u7c7b\u578b\u63cf\u8ff0\nDict&#91;str, Any]\nDict\uff1a\u5b57\u5178\u7c7b\u578b\uff08\u82b1\u62ec\u53f7 {} \u90a3\u79cd\uff09\n&#91;str, Any]\uff1a\u65b9\u62ec\u53f7\u91cc\u7684\u5185\u5bb9\u662f\u8bf4\u660e\u8fd9\u4e2a\u5b57\u5178\u7684\u952e\u503c\u5bf9\u7c7b\u578b\n  str\uff1a\u952e\uff08key\uff09\u662f\u5b57\u7b26\u4e32\u7c7b\u578b\n  Any\uff1a\u503c\uff08value\uff09\u53ef\u4ee5\u662f\u4efb\u4f55\u7c7b\u578b\n\n\u7c7b\u6bd4\u7406\u89e3\uff1a\n\u60f3\u8c61\u4e00\u4e0b\uff0c\u4f60\u8981\u53bb\u8d85\u5e02\u4e70\u4e1c\u897f\uff1a\ndef \u4e70\u4e1c\u897f(\u94b1: int) -> \u8d2d\u7269\u888b:\n    # \u7528\u94b1\u4e70\u4e1c\u897f\n    return \u88c5\u6ee1\u4e1c\u897f\u7684\u8d2d\u7269\u888b\n\u94b1: int\uff1a\u544a\u8bc9\u4f60\u9700\u8981\u5e26\u6574\u6570\uff08\u6bd4\u5982100\u5143\uff09\n-> \u8d2d\u7269\u888b\uff1a\u544a\u8bc9\u4f60\u51fd\u6570\u4f1a\u8fd4\u56de\u4e00\u4e2a\u8d2d\u7269\u888b\n'''\n#\u8fd9\u4e2a\u4ee3\u7801\u4e2d\u7684\u5176\u4ed6\u4f8b\u5b50\uff1a\n'''\n\u4f8b1\uff1a\u6709\u53c2\u6570\u7684\u7c7b\u578b\u63d0\u793a\n\nasync def execute_action(self, action: Dict&#91;str, Any]) -> ActionResult:\n\u8f93\u5165\uff1aaction: Dict&#91;str, Any]\n  \u53c2\u6570\u540d\u53eb action\n  \u7c7b\u578b\u662f\u5b57\u5178\uff0c\u952e\u662f\u5b57\u7b26\u4e32\uff0c\u503c\u53ef\u4ee5\u662f\u4efb\u4f55\u7c7b\u578b\n\u8f93\u51fa\uff1a-> ActionResult\n  \u4f1a\u8fd4\u56de\u4e00\u4e2a ActionResult \u7c7b\u578b\u7684\u5bf9\u8c61\n\n\u4f8b2\uff1a\u591a\u4e2a\u53c2\u6570\u7684\u7c7b\u578b\u63d0\u793a\ndef _parse_coordinates(self, x, y) -> Tuple&#91;int, int]:\n\u8f93\u51fa\uff1a-> Tuple&#91;int, int]\n \u4f1a\u8fd4\u56de\u4e00\u4e2a\u5143\u7ec4\uff0c\u5305\u542b\u4e24\u4e2a\u6574\u6570\n\n\u4f8b3\uff1a\u53ef\u9009\u7684\u7c7b\u578b\ndata: Optional&#91;Dict&#91;str, Any]] = None\nOptional&#91;...] \u8868\u793a\"\u53ef\u80fd\u662f\u8fd9\u4e2a\u7c7b\u578b\uff0c\u4e5f\u53ef\u80fd\u662fNone\"\n\u6240\u4ee5\u8fd9\u91cc\u7684\u610f\u601d\u662f\uff1adata \u53ef\u80fd\u662f\u5b57\u5178\uff0c\u4e5f\u53ef\u80fd\u662f None\n'''\n#\u4e3a\u4ec0\u4e48\u8981\u6709\u8fd9\u4e2a\uff1f\n'''\n\u597d\u59041\uff1a\u50cf\u8bf4\u660e\u4e66\u4e00\u6837\n\n# \u6ca1\u6709\u7c7b\u578b\u63d0\u793a\uff0c\u4f60\u4e0d\u77e5\u9053\u8be5\u4f20\u4ec0\u4e48\ndef \u8ba1\u7b97(\u53c2\u6570):\n    pass\n\n# \u6709\u7c7b\u578b\u63d0\u793a\uff0c\u5f88\u6e05\u695a\u8be5\u4f20\u4ec0\u4e48\ndef \u8ba1\u7b97(\u53c2\u6570: int) -> float:\n    \"\"\"\n    \u53c2\u6570\u5e94\u8be5\u4f20\u6574\u6570\n    \u4f1a\u8fd4\u56de\u5c0f\u6570\n    \"\"\"\n    pass\n    \n\u597d\u59042\uff1aIDE\uff08\u7f16\u7a0b\u5de5\u5177\uff09\u80fd\u5e2e\u4f60\n\u7f16\u8f91\u5668\u4f1a\u63d0\u793a\u4f60\u8be5\u4f20\u4ec0\u4e48\u7c7b\u578b\u7684\u53c2\u6570\n\u5982\u679c\u4f60\u4f20\u9519\u4e86\u7c7b\u578b\uff0c\u7f16\u8f91\u5668\u4f1a\u8b66\u544a\u4f60\n\u4ee3\u7801\u8865\u5168\u4f1a\u66f4\u51c6\u786e\n\n\u597d\u59043\uff1a\u8ba9\u522b\u4eba\u66f4\u5bb9\u6613\u7406\u89e3\u4f60\u7684\u4ee3\u7801\n\u5c31\u50cf\u4f60\u5199\u7684\"\u4f7f\u7528\u8bf4\u660e\u4e66\"\n'''\n#\u5b9e\u9645\u8fd0\u884c\u65f6\u4f1a\u53d1\u751f\u4ec0\u4e48\n#\u91cd\u8981\uff1a\u8fd9\u4e9b\u7c7b\u578b\u63d0\u793a\u53ea\u662f\u63d0\u793a\uff01Python\u5728\u8fd0\u884c\u65f6\u4f1a\u5ffd\u7565\u5b83\u4eec\uff01\n'''\n\u770b\u8fd9\u4e2a\u4f8b\u5b50\uff1a\ndef \u52a0\u6cd5(a: int, b: int) -> int:\n    return a + b\n\n# \u4e0b\u9762\u8fd9\u4e9b\u90fd\u80fd\u8fd0\u884c\uff0c\u4e0d\u4f1a\u62a5\u9519\uff01\nprint(\u52a0\u6cd5(1, 2))        # \u2705 \u6b63\u786e\uff1a3\nprint(\u52a0\u6cd5(\"hello\", \" world\"))  # \u2705 \u4e5f\u80fd\u8fd0\u884c\uff1a\"hello world\"\nprint(\u52a0\u6cd5(&#91;1], &#91;2]))    # \u2705 \u4e5f\u80fd\u8fd0\u884c\uff1a&#91;1, 2]\n\n\u7c7b\u578b\u63d0\u793a\u53ea\u662f\u7ed9\u7a0b\u5e8f\u5458\u548c\u7f16\u7a0b\u5de5\u5177\u770b\u7684\uff0cPython\u89e3\u91ca\u5668\u4e0d\u68c0\u67e5\u5b83\u4eec\u3002\n'''\n#\u8fd9\u4e2a\u6587\u4ef6\u4e2d\u51fa\u73b0\u7684\u6240\u6709\u7c7b\u578b\u63d0\u793a\uff1a\n'''\n# \u51fd\u6570\u53c2\u6570\u7684\u7c7b\u578b\u63d0\u793a\ndef \u51fd\u6570\u540d(\u53c2\u6570: \u7c7b\u578b, \u53c2\u65702: \u7c7b\u578b)\n\n# \u51fd\u6570\u8fd4\u56de\u503c\u7684\u7c7b\u578b\u63d0\u793a  \ndef \u51fd\u6570\u540d(...) -> \u8fd4\u56de\u7c7b\u578b:\n\n# \u53d8\u91cf\u7c7b\u578b\u63d0\u793a\n\u53d8\u91cf\u540d: \u7c7b\u578b = \u521d\u59cb\u503c\n'''\n#\u5e38\u89c1\u7c7b\u578b\uff1a\n'''\nDict&#91;str, Any]      # \u5b57\u5178\uff0c\u952e\u662f\u5b57\u7b26\u4e32\uff0c\u503c\u662f\u4efb\u4f55\u7c7b\u578b\nOptional&#91;...]       # \u53ef\u80fd\u662f...\uff0c\u4e5f\u53ef\u80fd\u662fNone\nTuple&#91;int, int]     # \u5305\u542b\u4e24\u4e2a\u6574\u6570\u7684\u5143\u7ec4\nList&#91;ActionResult]  # ActionResult\u7c7b\u578b\u7684\u5217\u8868\nbool                # \u5e03\u5c14\u503c\uff08True\/False\uff09\nstr                 # \u5b57\u7b26\u4e32\nint                 # \u6574\u6570\nfloat               # \u5c0f\u6570\n'''\n#\u6765\uff0c\u590d\u4e60\u4e00\u4e0b\n'''\ndef \u83b7\u53d6\u5b66\u751f\u4fe1\u606f(\u5b66\u53f7: str) -> Optional&#91;Dict&#91;str, Any]]:\n    \"\"\"\n    \u8f93\u5165\u5b66\u53f7\uff08\u5b57\u7b26\u4e32\uff09\n    \u8fd4\u56de\u5b66\u751f\u4fe1\u606f\uff08\u5b57\u5178\uff0c\u53ef\u80fd\u662fNone\uff09\n    \"\"\"\n    \n\u7b54\u6848\uff1a\n\u51fd\u6570\u540d\uff1a\u83b7\u53d6\u5b66\u751f\u4fe1\u606f\n\u9700\u8981\u4f20\u4e00\u4e2a\u53c2\u6570\uff1a\u5b66\u53f7\uff0c\u7c7b\u578b\u662f\u5b57\u7b26\u4e32\n\u8fd4\u56de\u503c\uff1a\u53ef\u80fd\u662f\u4e00\u4e2a\u5b57\u5178\uff08\u952e\u662f\u5b57\u7b26\u4e32\uff0c\u503c\u662f\u4efb\u4f55\u7c7b\u578b\uff09\uff0c\u4e5f\u53ef\u80fd\u662f None\n'''\n#\u603b\u7ed3\uff1a\n#1.-> \u662f\u7c7b\u578b\u63d0\u793a\u7bad\u5934\uff0c\u8868\u793a\u51fd\u6570\u8fd4\u56de\u4ec0\u4e48\u7c7b\u578b\n#2.\u5b83\u53ea\u662f\u63d0\u793a\uff0cPython\u8fd0\u884c\u65f6\u4e0d\u68c0\u67e5\n#3.\u5b83\u8ba9\u4ee3\u7801\u66f4\u6e05\u6670\uff0c\u66f4\u5bb9\u6613\u7406\u89e3\n#4.\u73b0\u4ee3Python\u7f16\u7a0b\u4e2d\u63a8\u8350\u4f7f\u7528<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">computer_use_adapter.py<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>\"\"\"\n\u7535\u8111\u63a7\u5236\u9002\u914d\u5668 - \u57fa\u4e8e\u535a\u5f08\u8bba\u7684ComputerUseAdapter\u5b9e\u73b0\n\u63d0\u4f9b\u9f20\u6807\u952e\u76d8\u63a7\u5236\u3001\u5c4f\u5e55\u622a\u56fe\u3001\u89c6\u89c9\u5206\u6790\u7b49\u6838\u5fc3\u529f\u80fd\n\"\"\"\n\nimport io           # \u8f93\u5165\u8f93\u51fa\u5de5\u5177\uff0c\u5904\u7406\u6570\u636e\u6d41\nimport time         # \u65f6\u95f4\u5de5\u5177\uff0c\u7528\u4e8e\u7b49\u5f85\u548c\u8ba1\u65f6\nimport platform     # \u7cfb\u7edf\u5e73\u53f0\u5de5\u5177\uff0c\u8bc6\u522b\u662fWindows\/Mac\/Linux\nimport logging      # \u65e5\u5fd7\u5de5\u5177\uff0c\u8bb0\u5f55\u8fd0\u884c\u8fc7\u7a0b\nfrom typing import Dict, Any, Optional, Tuple  # \u7c7b\u578b\u63d0\u793a\nimport sys          # \u7cfb\u7edf\u5de5\u5177\uff0c\u7ba1\u7406Python\u8fd0\u884c\u73af\u5883\n#\u7279\u6b8a\u5904\u7406 - PIL\u5e93\nif 'nagaagent_core.vendors.pil' in sys.modules:\n    del sys.modules&#91;'nagaagent_core.vendors.pil']\nfrom PIL import Image\n'''\n\u8fd9\u662f\u4ec0\u4e48\u610f\u601d\uff1f\n\u68c0\u67e5\u662f\u5426\u5bfc\u5165\u8fc7\u67d0\u4e2a\u7279\u5b9a\u7684PIL\u5e93\n\u5982\u679c\u5bfc\u5165\u8fc7\uff0c\u5c31\u5220\u9664\u5b83\uff08\u907f\u514d\u51b2\u7a81\uff09\n\u7136\u540e\u91cd\u65b0\u5bfc\u5165\u6807\u51c6\u7684PIL\u5e93\uff08Python\u7684\u56fe\u50cf\u5904\u7406\u5e93\uff09\n\u8fd9\u5c31\u50cf\uff1a\u5148\u68c0\u67e5\u4e66\u5305\u91cc\u6709\u6ca1\u6709\u65e7\u7248\u672c\u7684\u8bfe\u672c\uff0c\u5982\u679c\u6709\u5c31\u5148\u62ff\u51fa\u6765\uff0c\u7136\u540e\u653e\u65b0\u7248\u672c\u7684\u8fdb\u53bb\u3002\n'''\nimport asyncio # \u5f02\u6b65\u7f16\u7a0b\u5de5\u5177\uff0c\u8ba9\u7a0b\u5e8f\u80fd\u540c\u65f6\u505a\u591a\u4ef6\u4e8b\n\n# \u5c1d\u8bd5\u5bfc\u5165\u4f9d\u8d56\u5305\uff08\u53ef\u80fd\u6ca1\u6709\u7684\u5de5\u5177\uff09\ntry:   #\u5c1d\u8bd5\u5bfc\u5165pyautogui\uff08\u63a7\u5236\u9f20\u6807\u952e\u76d8\u7684\u6838\u5fc3\u5de5\u5177\uff09\n    import nagaagent_core.vendors.pyautogui as pyautogui\n    PYAUTOGUI_AVAILABLE = True\nexcept ImportError:  # \u5c31\u6807\u8bb0\u4e3a\u4e0d\u53ef\u7528\n    PYAUTOGUI_AVAILABLE = False\n    pyautogui = None\n#\u5982\u679cpyautogui\u5b89\u88c5\u4e86\uff0c\u5c31\u80fd\u63a7\u5236\u9f20\u6807\u952e\u76d8,\u5982\u679c\u6ca1\u5b89\u88c5\uff0c\u7a0b\u5e8f\u4e0d\u4f1a\u5d29\u6e83\uff0c\u53ea\u662f\u76f8\u5173\u529f\u80fd\u4e0d\u80fd\u7528\n\ntry:  #\u5c1d\u8bd5\u5bfc\u5165GUI Agents\uff08AI\u89c6\u89c9\u5206\u6790\u5de5\u5177\uff09\n    from gui_agents.s2_5.agents.grounding import OSWorldACI  #OSWorldACI\uff1a\u64cd\u4f5c\u7cfb\u7edf\u4e16\u754c\u63a5\u53e3\uff0c\u53ef\u80fd\u7528\u4e8e\u7406\u89e3\u5c4f\u5e55\u5185\u5bb9\n    from gui_agents.s2_5.agents.agent_s import AgentS2_5   #AgentS2_5\uff1a\u4e00\u4e2aAI\u4ee3\u7406\uff0c\u53ef\u80fd\u7528\u4e8e\u8bc6\u522b\u5c4f\u5e55\u4e0a\u7684\u5143\u7d20\n    GUI_AGENTS_AVAILABLE = True\nexcept ImportError:    #\u91cd\u8981\u53d8\u91cf\uff08\u6807\u8bb0\u5de5\u5177\u662f\u5426\u53ef\u7528\uff09\n    GUI_AGENTS_AVAILABLE = False\n    OSWorldACI = None\n    AgentS2_5 = None\n'''\n\u8fd9\u4e9b\u6807\u8bb0\u544a\u8bc9\u7a0b\u5e8f\uff1a\n\u54ea\u4e9b\u529f\u80fd\u53ef\u4ee5\u4f7f\u7528\n\u54ea\u4e9b\u529f\u80fd\u56e0\u4e3a\u7f3a\u5c11\u5de5\u5177\u800c\u4e0d\u80fd\u7528\n'''\n\n# \u914d\u7f6e\u65e5\u5fd7\nlogger = logging.getLogger(__name__)\n\nclass ComputerUseAdapter:\n    \"\"\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\uff0c\u57fa\u4e8e\u535a\u5f08\u8bba\u7684\u5b9e\u73b0\"\"\"\n    \n    #\u7c7b\u7684\u521d\u59cb\u5316 __init__\n    def __init__(self):\n        \"\"\"\u521d\u59cb\u5316\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\"\"\"\n        self.last_error: Optional&#91;str] = None  # \u8bb0\u5f55\u6700\u540e\u9519\u8bef\n        self.agent = None                      # AI\u4ee3\u7406\n        self.grounding_agent = None            # \u57fa\u7840\u4ee3\u7406\n        self.init_ok = False                   # \u662f\u5426\u521d\u59cb\u5316\u6210\u529f\n        \n        # \u8bbe\u7f6eDPI\u611f\u77e5\uff08Windows\uff09\n        self._setup_dpi_awareness()\n        \n        # \u521d\u59cb\u5316\u5c4f\u5e55\u5c3a\u5bf8 - \u57fa\u4e8e\u535a\u5f08\u8bba\u7684\u5b9e\u73b0\n        self.screen_width = 1920  # \u9ed8\u8ba4\u503c\n        self.screen_height = 1080  # \u9ed8\u8ba4\u503c\n        self.scaled_width = 1920\n        self.scaled_height = 1080\n        self.scale_x = 1.0\n        self.scale_y = 1.0\n        #\u7269\u7406\u5c3a\u5bf8\uff1a\u7535\u8111\u5c4f\u5e55\u7684\u5b9e\u9645\u5206\u8fa8\u7387,\u903b\u8f91\u5c3a\u5bf8\uff1a\u7edf\u4e00\u7f29\u653e\u52301920x1080\u7684\u6807\u51c6\u5c3a\u5bf8\n        #\u4e3a\u4ec0\u4e48?\u4e0d\u540c\u7535\u8111\u7684\u5c4f\u5e55\u5927\u5c0f\u4e0d\u540c,\u7edf\u4e00\u6807\u51c6\u8ba9AI\u66f4\u5bb9\u6613\u8bc6\u522b,\u5c31\u50cf\u628a\u6240\u6709\u7167\u7247\u90fd\u7f29\u653e\u5230\u7edf\u4e00\u5927\u5c0f\n        \n        if PYAUTOGUI_AVAILABLE:  #\u7528pyautogui\u83b7\u53d6\u5b9e\u9645\u5c4f\u5e55\u5927\u5c0f\n            try:\n                # \u52a8\u6001\u83b7\u53d6\u5b9e\u9645\u5c4f\u5e55\u5c3a\u5bf8\n                self.screen_width, self.screen_height = pyautogui.size()\n                logger.info(f\"\u5b9e\u9645\u5c4f\u5e55\u5c3a\u5bf8: {self.screen_width}x{self.screen_height}\")\n                \n                # \u8ba1\u7b97\u7f29\u653e\u5c3a\u5bf8\uff08\u53c2\u8003\u535a\u5f08\u8bba\u7684scale_screen_dimensions\u51fd\u6570\uff09\n                self.scaled_width, self.scaled_height = self._scale_screen_dimensions( #\u8c03\u7528_scale_screen_dimensions\u8ba1\u7b97\u6807\u51c6\u5c3a\u5bf8\n                    self.screen_width, self.screen_height, max_dim_size=1920\n                )\n                \n                # \u8ba1\u7b97\u7f29\u653e\u56e0\u5b50\uff08\u903b\u8f91\u5750\u6807 -> \u7269\u7406\u5750\u6807\uff09,\u8ba1\u7b97\u7f29\u653e\u6bd4\u4f8b\uff08\u5b9e\u9645\u2192\u6807\u51c6\uff09\n                self.scale_x = self.screen_width \/ max(1, self.scaled_width)\n                self.scale_y = self.screen_height \/ max(1, self.scaled_height)\n                \n            except Exception as e:\n                logger.warning(f\"\u83b7\u53d6\u5c4f\u5e55\u5c3a\u5bf8\u5931\u8d25: {e}\")\n                # \u4f7f\u7528\u9ed8\u8ba4\u503c\n        \n        # \u521d\u59cb\u5316\u7ec4\u4ef6\n        self._init_components()\n    \n    def _setup_dpi_awareness(self):    #\u6709\u4e9b\u7535\u8111\u5c4f\u5e55\u663e\u793a\u6709\u7f29\u653e\uff08\u6bd4\u5982150%\u663e\u793a\uff09\u544a\u8bc9Windows\u7cfb\u7edf\uff0c\u8fd9\u4e2a\u7a0b\u5e8f\u9700\u8981\u7cbe\u786e\u7684\u5750\u6807,\u8c03\u7528Windows\u7684API\u51fd\u6570\n        \"\"\"\u8bbe\u7f6eDPI\u611f\u77e5\uff0c\u63d0\u9ad8\u5750\u6807\u7cbe\u5ea6\"\"\"\n        try:\n            if platform.system().lower() == \"windows\":   # \u53ea\u5728Windows\u7cfb\u7edf\u4e0a\u505a\n                import ctypes\n                try:\n                    ctypes.windll.shcore.SetProcessDpiAwareness(2)\n                except Exception:\n                    try:\n                        ctypes.windll.user32.SetProcessDPIAware()\n                    except Exception:\n                        pass\n        except Exception:\n            pass\n    \n    #\u5c4f\u5e55\u7f29\u653e\u51fd\u6570\uff08\u5c3a\u5b50\u8f6c\u6362\u516c\u5f0f\uff09\n    def _scale_screen_dimensions(self, width: int, height: int, max_dim_size: int = 1920) -> Tuple&#91;int, int]:\n        \"\"\"\u7f29\u653e\u5c4f\u5e55\u5c3a\u5bf8\uff0c\u57fa\u4e8e\u535a\u5f08\u8bba\u7684\u5b9e\u73b0\"\"\"\n        scale_factor = min(max_dim_size \/ width, max_dim_size \/ height)\n        safe_width = int(width * scale_factor)\n        safe_height = int(height * scale_factor)\n        logger.info(f\"\u5c4f\u5e55\u7f29\u653e: {width}x{height} -> {safe_width}x{safe_height}, \u7f29\u653e\u56e0\u5b50: {scale_factor:.2f}\")\n        return safe_width, safe_height\n    \n    def _init_components(self):\n        \"\"\"\u521d\u59cb\u5316\u6838\u5fc3\u7ec4\u4ef6\uff08\u542f\u52a8\u5404\u4e2a\u90e8\u4ef6\uff09\"\"\"\n        try:\n            if not PYAUTOGUI_AVAILABLE:       # \u5982\u679c\u6ca1\u6709pyautogui\uff0c\u5c31\u62a5\u9519\u8fd4\u56de\n                self.last_error = \"pyautogui\u672a\u5b89\u88c5\"\n                logger.error(\"pyautogui\u672a\u5b89\u88c5\uff0c\u7535\u8111\u63a7\u5236\u529f\u80fd\u4e0d\u53ef\u7528\")\n                return\n            \n            if not GUI_AGENTS_AVAILABLE:      # \u5982\u679c\u6ca1\u6709GUI\u4ee3\u7406\uff0c\u5c31\u62a5\u9519\u8fd4\u56de\n                self.last_error = \"gui-agents\u672a\u5b89\u88c5\"\n                logger.error(\"gui-agents\u672a\u5b89\u88c5\uff0c\u9ad8\u7ea7\u529f\u80fd\u4e0d\u53ef\u7528\")\n                return\n            \n            # \u521d\u59cb\u5316GUI\u4ee3\u7406\n            self._init_gui_agents()\n            self.init_ok = True\n            logger.info(\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u521d\u59cb\u5316\u6210\u529f\")\n            \n        except Exception as e:\n            self.last_error = str(e)\n            logger.error(f\"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u521d\u59cb\u5316\u5931\u8d25: {e}\")\n    \n    def _init_gui_agents(self):\n        \"\"\"\u521d\u59cb\u5316GUI\u4ee3\u7406\"\"\"\n        try:\n            # \u8fd9\u91cc\u53ef\u4ee5\u6dfb\u52a0GUI\u4ee3\u7406\u7684\u521d\u59cb\u5316\u903b\u8f91\n            # \u6682\u65f6\u4f7f\u7528\u57fa\u7840\u529f\u80fd\n            logger.info(\"GUI\u4ee3\u7406\u521d\u59cb\u5316\u5b8c\u6210\")\n        except Exception as e:\n            logger.warning(f\"GUI\u4ee3\u7406\u521d\u59cb\u5316\u5931\u8d25: {e}\")\n    \n    def is_available(self) -> Dict&#91;str, Any]:\n        \"\"\"\u68c0\u67e5\u7535\u8111\u63a7\u5236\u529f\u80fd\u662f\u5426\u53ef\u7528\"\"\"\n        ok = True\n        reasons = &#91;]\n        \n        if not PYAUTOGUI_AVAILABLE:\n            ok = False\n            reasons.append(\"pyautogui\u672a\u5b89\u88c5\")\n        \n        if not self.init_ok:\n            ok = False\n            msg = \"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"\n            if self.last_error:\n                msg += f\": {self.last_error}\"\n            reasons.append(msg)\n        \n        return {\n            \"enabled\": True,\n            \"ready\": ok,\n            \"reasons\": reasons,\n            \"screen_info\": {\n                \"physical_size\": f\"{self.screen_width}x{self.screen_height}\",\n                \"scaled_size\": f\"{self.scaled_width}x{self.scaled_height}\",\n                \"scale_factors\": f\"x={self.scale_x:.2f}, y={self.scale_y:.2f}\"\n            },\n            \"platform\": platform.system()\n        }\n    \n    async def take_screenshot(self) -> Optional&#91;bytes]:\n        \"\"\"\u622a\u53d6\u5c4f\u5e55\u622a\u56fe\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return None\n        \n        try:\n            screenshot = pyautogui.screenshot()\n            # \u57fa\u4e8e\u535a\u5f08\u8bba\u7684\u5b9e\u73b0\uff1a\u5c06\u622a\u56fe\u7f29\u653e\u5230\u903b\u8f91\u5c3a\u5bf8\n            screenshot = screenshot.resize((self.scaled_width, self.scaled_height), Image.LANCZOS)\n            buf = io.BytesIO()\n            screenshot.save(buf, format=\"PNG\")\n            return buf.getvalue()\n        except Exception as e:\n            logger.error(f\"\u622a\u53d6\u5c4f\u5e55\u622a\u56fe\u5931\u8d25: {e}\")\n            return None\n    \n    def _scale_coordinates(self, x: int, y: int) -> Tuple&#91;int, int]:\n        \"\"\"\u7f29\u653e\u5750\u6807\uff0c\u5c06\u903b\u8f91\u5750\u6807\u8f6c\u6362\u4e3a\u7269\u7406\u5750\u6807\"\"\"\n        scaled_x = int(round(x * self.scale_x))\n        scaled_y = int(round(y * self.scale_y))\n        return scaled_x, scaled_y\n    \n    def _create_scaled_pyautogui(self):\n        \"\"\"\u521b\u5efa\u7f29\u653e\u7248\u672c\u7684pyautogui\u4ee3\u7406\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return None\n        \n        class _ScaledPyAutoGUI:\n            \"\"\"\n            \u8f7b\u91cf\u7ea7\u4ee3\u7406\uff0c\u5c06\u903b\u8f91\u5750\u6807\u7a7a\u95f4\u7f29\u653e\u5230\u7269\u7406\u5c4f\u5e55\u5750\u6807\n            \u652f\u6301\u591a\u79cd\u5750\u6807\u683c\u5f0f\n            \"\"\"\n            def __init__(self, backend, scale_x: float, scale_y: float):\n                self._backend = backend\n                self._scale_x = scale_x\n                self._scale_y = scale_y\n\n            def __getattr__(self, name):\n                # \u56de\u9000\u5230\u6240\u6709\u5176\u4ed6\u5c5e\u6027\/\u65b9\u6cd5\n                return getattr(self._backend, name)\n\n            def _scale_xy_from_args(self, args, kwargs):\n                \"\"\"\u4ece\u53c2\u6570\u4e2d\u7f29\u653ex,y\u5750\u6807\uff0c\u652f\u6301\u591a\u79cd\u5750\u6807\u683c\u5f0f\"\"\"\n                # \u5904\u7406 (x, y) \u683c\u5f0f\n                if len(args) >= 2 and isinstance(args&#91;0], (int, float)) and isinstance(args&#91;1], (int, float)):\n                    x = int(round(args&#91;0] * self._scale_x))\n                    y = int(round(args&#91;1] * self._scale_y))\n                    args = (x, y) + tuple(args&#91;2:])\n                # \u5904\u7406 ((x, y),) \u683c\u5f0f\n                elif len(args) >= 1 and isinstance(args&#91;0], (tuple, list)) and len(args&#91;0]) == 2:\n                    x_raw, y_raw = args&#91;0]\n                    if isinstance(x_raw, (int, float)) and isinstance(y_raw, (int, float)):\n                        x = int(round(x_raw * self._scale_x))\n                        y = int(round(y_raw * self._scale_y))\n                        args = ((x, y),) + tuple(args&#91;1:])\n                else:\n                    # \u5904\u7406kwargs\u683c\u5f0f\n                    if 'x' in kwargs and 'y' in kwargs and isinstance(kwargs&#91;'x'], (int, float)) and isinstance(kwargs&#91;'y'], (int, float)):\n                        kwargs = dict(kwargs)\n                        kwargs&#91;'x'] = int(round(kwargs&#91;'x'] * self._scale_x))\n                        kwargs&#91;'y'] = int(round(kwargs&#91;'y'] * self._scale_y))\n                return args, kwargs\n\n            # \u5305\u88c5\u7684\u7edd\u5bf9\u5750\u6807\u9f20\u6807API - \u81ea\u52a8\u5750\u6807\u7f29\u653e\n            def moveTo(self, *args, **kwargs):\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.moveTo(*args, **kwargs)\n\n            def click(self, *args, **kwargs):\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.click(*args, **kwargs)\n\n            def doubleClick(self, *args, **kwargs):\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.doubleClick(*args, **kwargs)\n\n            def rightClick(self, *args, **kwargs):\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.rightClick(*args, **kwargs)\n\n            def dragTo(self, *args, **kwargs):\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.dragTo(*args, **kwargs)\n            \n            def scroll(self, *args, **kwargs):\n                \"\"\"\u6eda\u52a8\u64cd\u4f5c\u4e5f\u652f\u6301\u5750\u6807\u7f29\u653e\"\"\"\n                args, kwargs = self._scale_xy_from_args(args, kwargs)\n                return self._backend.scroll(*args, **kwargs)\n        \n        return _ScaledPyAutoGUI(pyautogui, self.scale_x, self.scale_y)\n    \n    async def click(self, x: int, y: int, button: str = 'left') -> bool:\n        \"\"\"\u70b9\u51fb\u6307\u5b9a\u5750\u6807\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return False\n        \n        try:\n            # \u7f29\u653e\u5750\u6807\n            scaled_x, scaled_y = self._scale_coordinates(x, y)\n            \n            if button == 'left':\n                pyautogui.click(scaled_x, scaled_y)\n            elif button == 'right':\n                pyautogui.rightClick(scaled_x, scaled_y)\n            elif button == 'middle':\n                pyautogui.middleClick(scaled_x, scaled_y)\n            else:\n                pyautogui.click(scaled_x, scaled_y)\n            \n            logger.info(f\"\u70b9\u51fb\u5750\u6807: \u903b\u8f91({x}, {y}) -> \u7269\u7406({scaled_x}, {scaled_y}), \u6309\u94ae: {button}\")\n            return True\n        except Exception as e:\n            logger.error(f\"\u70b9\u51fb\u64cd\u4f5c\u5931\u8d25: {e}\")\n            return False\n    \n    async def type_text(self, text: str, interval: float = 0.1) -> bool:\n        \"\"\"\u8f93\u5165\u6587\u672c\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return False\n        \n        try:\n            pyautogui.typewrite(text, interval=interval)\n            logger.info(f\"\u8f93\u5165\u6587\u672c: {text}\")\n            return True\n        except Exception as e:\n            logger.error(f\"\u8f93\u5165\u6587\u672c\u5931\u8d25: {e}\")\n            return False\n    \n    async def press_key(self, key: str) -> bool:\n        \"\"\"\u6309\u4e0b\u6307\u5b9a\u6309\u952e\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return False\n        \n        try:\n            pyautogui.press(key)\n            logger.info(f\"\u6309\u4e0b\u6309\u952e: {key}\")\n            return True\n        except Exception as e:\n            logger.error(f\"\u6309\u952e\u64cd\u4f5c\u5931\u8d25: {e}\")\n            return False\n    \n    async def scroll(self, x: int, y: int, clicks: int) -> bool:\n        \"\"\"\u6eda\u52a8\u9f20\u6807\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return False\n        \n        try:\n            # \u7f29\u653e\u5750\u6807\n            scaled_x, scaled_y = self._scale_coordinates(x, y)\n            pyautogui.scroll(clicks, scaled_x, scaled_y)\n            logger.info(f\"\u6eda\u52a8: \u903b\u8f91({x}, {y}) -> \u7269\u7406({scaled_x}, {scaled_y}), \u6eda\u52a8\u91cf: {clicks}\")\n            return True\n        except Exception as e:\n            logger.error(f\"\u6eda\u52a8\u64cd\u4f5c\u5931\u8d25: {e}\")\n            return False\n    \n    async def drag_to(self, start_x: int, start_y: int, end_x: int, end_y: int, duration: float = 1.0) -> bool:\n        \"\"\"\u62d6\u62fd\u64cd\u4f5c\"\"\"\n        if not PYAUTOGUI_AVAILABLE:\n            return False\n        \n        try:\n            # \u7f29\u653e\u5750\u6807\n            scaled_start_x, scaled_start_y = self._scale_coordinates(start_x, start_y)\n            scaled_end_x, scaled_end_y = self._scale_coordinates(end_x, end_y)\n            \n            pyautogui.dragTo(scaled_end_x, scaled_end_y, duration=duration)\n            logger.info(f\"\u62d6\u62fd: \u903b\u8f91({start_x}, {start_y})->({end_x}, {end_y}) -> \u7269\u7406({scaled_start_x}, {scaled_start_y})->({scaled_end_x}, {scaled_end_y})\")\n            return True\n        except Exception as e:\n            logger.error(f\"\u62d6\u62fd\u64cd\u4f5c\u5931\u8d25: {e}\")\n            return False\n    \n    async def find_element_by_text(self, text: str, screenshot: Optional&#91;bytes] = None) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\u901a\u8fc7\u6587\u672c\u67e5\u627e\u5143\u7d20\u4f4d\u7f6e\"\"\"\n        # \u8fd9\u91cc\u53ef\u4ee5\u96c6\u6210OCR\u529f\u80fd\n        # \u6682\u65f6\u8fd4\u56deNone\uff0c\u540e\u7eed\u5b9e\u73b0\n        logger.info(f\"\u67e5\u627e\u6587\u672c\u5143\u7d20: {text}\")\n        return None\n    \n    async def find_element_by_image(self, image_path: str, screenshot: Optional&#91;bytes] = None) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\u901a\u8fc7\u56fe\u50cf\u67e5\u627e\u5143\u7d20\u4f4d\u7f6e\"\"\"\n        # \u8fd9\u91cc\u53ef\u4ee5\u96c6\u6210\u56fe\u50cf\u5339\u914d\u529f\u80fd\n        # \u6682\u65f6\u8fd4\u56deNone\uff0c\u540e\u7eed\u5b9e\u73b0\n        logger.info(f\"\u67e5\u627e\u56fe\u50cf\u5143\u7d20: {image_path}\")\n        return None\n    \n    async def execute_instruction(self, instruction: str) -> Dict&#91;str, Any]:\n        \"\"\"\u6267\u884c\u7535\u8111\u63a7\u5236\u6307\u4ee4\"\"\"\n        try:\n            logger.info(f\"\u6267\u884c\u6307\u4ee4: {instruction}\")\n            \n            # \u89e3\u6790\u6307\u4ee4\n            action = self._parse_instruction(instruction)\n            \n            if not action:\n                return {\n                    \"success\": False,\n                    \"error\": \"\u65e0\u6cd5\u89e3\u6790\u6307\u4ee4\",\n                    \"instruction\": instruction\n                }\n            \n            # \u6267\u884c\u52a8\u4f5c\n            result = await self._execute_action(action)\n            \n            return {\n                \"success\": result,\n                \"instruction\": instruction,\n                \"action\": action\n            }\n            \n        except Exception as e:\n            logger.error(f\"\u6267\u884c\u6307\u4ee4\u5931\u8d25: {e}\")\n            return {\n                \"success\": False,\n                \"error\": str(e),\n                \"instruction\": instruction\n            }\n    \n    async def run_instruction(self, instruction: str, max_iterations: int = 15) -> Dict&#91;str, Any]:\n        \"\"\"\u6267\u884c\u81ea\u7136\u8bed\u8a00\u6307\u4ee4\"\"\"\n        if not self.init_ok:\n            return {\"success\": False, \"error\": \"\u7535\u8111\u63a7\u5236\u9002\u914d\u5668\u672a\u521d\u59cb\u5316\"}\n        \n        try:\n            obs = {}\n            traj = \"\u4efb\u52a1:\\n\" + instruction\n            \n            for iteration in range(max_iterations):\n                logger.info(f\"\u6267\u884c\u8fed\u4ee3 {iteration + 1}\/{max_iterations}\")\n                \n                # \u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\n                screenshot = await self.take_screenshot()\n                if not screenshot:\n                    return {\"success\": False, \"error\": \"\u65e0\u6cd5\u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\"}\n                \n                obs&#91;\"screenshot\"] = screenshot\n                \n                # \u8fd9\u91cc\u5e94\u8be5\u8c03\u7528AI\u6a21\u578b\u6765\u751f\u6210\u4e0b\u4e00\u6b65\u52a8\u4f5c\n                # \u6682\u65f6\u4f7f\u7528\u7b80\u5355\u7684\u6a21\u62df\u903b\u8f91\n                if iteration == 0:\n                    # \u6a21\u62dfAI\u751f\u6210\u7684\u4ee3\u7801\n                    code = f\"# \u6267\u884c\u4efb\u52a1: {instruction}\\nprint('\u4efb\u52a1\u5f00\u59cb\u6267\u884c')\"\n                else:\n                    code = \"print('\u4efb\u52a1\u5b8c\u6210')\"\n                \n                logger.info(f\"\u6267\u884c\u4ee3\u7801: {code}\")\n                \n                # \u68c0\u67e5\u4efb\u52a1\u5b8c\u6210\u6761\u4ef6\n                if \"\u5b8c\u6210\" in code or \"done\" in code.lower():\n                    logger.info(\"\u4efb\u52a1\u5b8c\u6210\")\n                    break\n                \n                if \"\u7b49\u5f85\" in code or \"wait\" in code.lower():\n                    await asyncio.sleep(3)\n                    continue\n                \n                if \"\u4e0b\u4e00\u6b65\" in code or \"next\" in code.lower():\n                    continue\n                \n                # \u6267\u884c\u4ee3\u7801\uff08\u6ce8\u5165\u7f29\u653e\u540e\u7684pyautogui\uff09\n                try:\n                    exec_env = globals().copy()\n                    if PYAUTOGUI_AVAILABLE and hasattr(self, 'scale_x') and hasattr(self, 'scale_y'):\n                        scaled_pyautogui = self._create_scaled_pyautogui()\n                        if scaled_pyautogui:\n                            exec_env&#91;'pyautogui'] = scaled_pyautogui\n                    \n                    exec(code, exec_env, exec_env)\n                    await asyncio.sleep(0.5)\n                    \n                except Exception as e:\n                    logger.error(f\"\u4ee3\u7801\u6267\u884c\u5931\u8d25: {e}\")\n                    continue\n                \n                # \u66f4\u65b0\u4efb\u52a1\u8f68\u8ff9\n                traj += f\"\\n\\n\u8fed\u4ee3 {iteration + 1}:\\n{code}\"\n            \n            return {\n                \"success\": True, \n                \"message\": \"\u4efb\u52a1\u6267\u884c\u5b8c\u6210\",\n                \"iterations\": iteration + 1,\n                \"trajectory\": traj\n            }\n            \n        except Exception as e:\n            logger.error(f\"\u4efb\u52a1\u6267\u884c\u5931\u8d25: {e}\")\n            return {\"success\": False, \"error\": str(e)}\n    \n    def normalize_coordinates(self, x: float, y: float, screen_width: int = None, screen_height: int = None) -> Tuple&#91;int, int]:\n        \"\"\"\n        \u5750\u6807\u6807\u51c6\u5316\n        \u5c06\u4efb\u610f\u5750\u6807\u8f6c\u6362\u4e3a0-1000\u8303\u56f4\u7684\u6807\u51c6\u5316\u5750\u6807\n        \"\"\"\n        if screen_width is None:\n            screen_width = self.screen_width\n        if screen_height is None:\n            screen_height = self.screen_height\n        \n        # \u6807\u51c6\u5316\u52300-1000\u8303\u56f4\n        normalized_x = int(round(x \/ screen_width * 1000))\n        normalized_y = int(round(y \/ screen_height * 1000))\n        \n        # \u786e\u4fdd\u5728\u6709\u6548\u8303\u56f4\u5185\n        normalized_x = max(0, min(1000, normalized_x))\n        normalized_y = max(0, min(1000, normalized_y))\n        \n        return normalized_x, normalized_y\n    \n    def denormalize_coordinates(self, normalized_x: int, normalized_y: int, \n                               screen_width: int = None, screen_height: int = None) -> Tuple&#91;int, int]:\n        \"\"\"\n        \u53cd\u6807\u51c6\u5316\u5750\u6807\uff0c\u5c060-1000\u8303\u56f4\u7684\u5750\u6807\u8f6c\u6362\u4e3a\u5b9e\u9645\u50cf\u7d20\u5750\u6807\n        \u5750\u6807\u53cd\u6807\u51c6\u5316\n        \"\"\"\n        if screen_width is None:\n            screen_width = self.screen_width\n        if screen_height is None:\n            screen_height = self.screen_height\n        \n        # \u4ece\u6807\u51c6\u5316\u5750\u6807\u8f6c\u6362\u4e3a\u5b9e\u9645\u50cf\u7d20\u5750\u6807\n        pixel_x = int(round(normalized_x \/ 1000.0 * screen_width))\n        pixel_y = int(round(normalized_y \/ 1000.0 * screen_height))\n        \n        return pixel_x, pixel_y\n    \n    async def click_with_ai_location(self, target_description: str, button: str = 'left') -> bool:\n        \"\"\"\n        \u4f7f\u7528AI\u5b9a\u4f4d\u8fdb\u884c\u70b9\u51fb\n        \u652f\u6301\u81ea\u7136\u8bed\u8a00\u63cf\u8ff0\u76ee\u6807\u5143\u7d20\n        \"\"\"\n        try:\n            # \u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\n            screenshot = await self.take_screenshot()\n            if not screenshot:\n                logger.error(\"\u65e0\u6cd5\u83b7\u53d6\u5c4f\u5e55\u622a\u56fe\")\n                return False\n            \n            # \u4f7f\u7528AI\u5b9a\u4f4d\u5143\u7d20\n            from .visual_analyzer import VisualAnalyzer\n            analyzer = VisualAnalyzer()\n            location = await analyzer.locate_element_with_ai(\n                target_description, \n                screenshot, \n                self.screen_width, \n                self.screen_height\n            )\n            \n            if location:\n                x, y = location\n                return await self.click(x, y, button)\n            else:\n                logger.error(f\"AI\u5b9a\u4f4d\u5931\u8d25: {target_description}\")\n                return False\n                \n        except Exception as e:\n            logger.error(f\"AI\u5b9a\u4f4d\u70b9\u51fb\u5931\u8d25: {e}\")\n            return False\n    \n    def get_coordinate_info(self) -> Dict&#91;str, Any]:\n        \"\"\"\u83b7\u53d6\u5750\u6807\u7cfb\u7edf\u4fe1\u606f\"\"\"\n        return {\n            \"screen_size\": f\"{self.screen_width}x{self.screen_height}\",\n            \"scaled_size\": f\"{self.scaled_width}x{self.scaled_height}\",\n            \"scale_factors\": f\"x={self.scale_x:.2f}, y={self.scale_y:.2f}\",\n            \"normalization_range\": \"0-1000\",\n            \"platform\": platform.system()\n        }\n    \n    def _parse_instruction(self, instruction: str) -> Optional&#91;Dict&#91;str, Any]]:\n        \"\"\"\u89e3\u6790\u6307\u4ee4\"\"\"\n        instruction = instruction.lower().strip()\n        \n        # \u7b80\u5355\u7684\u6307\u4ee4\u89e3\u6790\n        if \"\u70b9\u51fb\" in instruction or \"click\" in instruction:\n            return {\"action\": \"click\", \"instruction\": instruction}\n        elif \"\u8f93\u5165\" in instruction or \"type\" in instruction:\n            return {\"action\": \"type\", \"instruction\": instruction}\n        elif \"\u622a\u56fe\" in instruction or \"screenshot\" in instruction:\n            return {\"action\": \"screenshot\", \"instruction\": instruction}\n        elif \"\u6eda\u52a8\" in instruction or \"scroll\" in instruction:\n            return {\"action\": \"scroll\", \"instruction\": instruction}\n        else:\n            return {\"action\": \"unknown\", \"instruction\": instruction}\n    \n    async def _execute_action(self, action: Dict&#91;str, Any]) -> bool:\n        \"\"\"\u6267\u884c\u5177\u4f53\u52a8\u4f5c\"\"\"\n        action_type = action.get(\"action\")\n        instruction = action.get(\"instruction\")\n        \n        if action_type == \"click\":\n            # \u8fd9\u91cc\u9700\u8981\u66f4\u667a\u80fd\u7684\u5750\u6807\u5b9a\u4f4d\n            # \u6682\u65f6\u4f7f\u7528\u5c4f\u5e55\u4e2d\u5fc3\n            x, y = self.screen_width \/\/ 2, self.screen_height \/\/ 2\n            return await self.click(x, y)\n        elif action_type == \"type\":\n            # \u8fd9\u91cc\u9700\u8981\u63d0\u53d6\u8981\u8f93\u5165\u7684\u6587\u672c\n            # \u6682\u65f6\u4f7f\u7528\u793a\u4f8b\u6587\u672c\n            return await self.type_text(\"Hello World\")\n        elif action_type == \"screenshot\":\n            screenshot = await self.take_screenshot()\n            return screenshot is not None\n        elif action_type == \"scroll\":\n            # \u8fd9\u91cc\u9700\u8981\u66f4\u667a\u80fd\u7684\u6eda\u52a8\u53c2\u6570\n            return await self.scroll(self.screen_width \/\/ 2, self.screen_height \/\/ 2, 3)\n        else:\n            logger.warning(f\"\u672a\u77e5\u52a8\u4f5c\u7c7b\u578b: {action_type}\")\n            return False\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">visual_analyzer.py<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>\"\"\"\n\u89c6\u89c9\u5206\u6790\u5668 - \u57fa\u4e8e\u535a\u5f08\u8bba\u7684\u89c6\u89c9\u8bc6\u522b\u529f\u80fd\n\u63d0\u4f9bOCR\u3001\u56fe\u50cf\u5339\u914d\u3001\u5143\u7d20\u5b9a\u4f4d\u3001AI\u5750\u6807\u5b9a\u4f4d\u7b49\u529f\u80fd\nAI\u5750\u6807\u5b9a\u4f4d\u7b97\u6cd5\u5347\u7ea7\n\"\"\"\n\nimport logging  # \u8bb0\u5f55\u65e5\u5fd7\u7684\uff0c\u50cf\u5199\u65e5\u8bb0\u4e00\u6837\u8bb0\u5f55\u7a0b\u5e8f\u8fd0\u884c\u60c5\u51b5\nfrom typing import Dict, Any, Optional, Tuple, List  # \u8fd9\u662fPython\u7684\u7c7b\u578b\u63d0\u793a\uff0c\u544a\u8bc9\u4f60\u51fd\u6570\u8fd4\u56de\u4ec0\u4e48\u7c7b\u578b\nimport sys  # \u7cfb\u7edf\u76f8\u5173\u7684\u529f\u80fd\n\n# \u4e0b\u9762\u8fd9\u4e24\u884c\u662f\u4e3a\u4e86\u9632\u6b62\u5bfc\u5165\u51b2\u7a81\nif 'nagaagent_core.vendors.pil' in sys.modules:\n    del sys.modules&#91;'nagaagent_core.vendors.pil']\n\nfrom PIL import Image  # \u5904\u7406\u56fe\u7247\u7684\u4e3b\u8981\u5de5\u5177\nimport io  # \u5904\u7406\u4e8c\u8fdb\u5236\u6570\u636e\uff08\u6bd4\u5982\u56fe\u7247\u6570\u636e\uff09\nimport base64  # \u628a\u56fe\u7247\u8f6c\u6362\u6210\u6587\u672c\u683c\u5f0f\nimport re  # \u6b63\u5219\u8868\u8fbe\u5f0f\uff0c\u5904\u7406\u6587\u672c\u6a21\u5f0f\u5339\u914d\nimport json  # \u5904\u7406JSON\u6570\u636e\u683c\u5f0f\n\n# \u914d\u7f6e\u65e5\u5fd7\nlogger = logging.getLogger(__name__)\n\nclass VisualAnalyzer:\n    \"\"\"\u89c6\u89c9\u5206\u6790\u5668\"\"\"\n    \n    def __init__(self):\n        \"\"\"\u521d\u59cb\u5316\u89c6\u89c9\u5206\u6790\u5668\"\"\"\n        self.ocr_available = False   # OCR\u529f\u80fd\u662f\u5426\u53ef\u7528\n        self.image_matching_available = False    # \u56fe\u50cf\u5339\u914d\u529f\u80fd\u662f\u5426\u53ef\u7528\n        self.ai_coordinate_available = False     # AI\u5b9a\u4f4d\u529f\u80fd\u662f\u5426\u53ef\u7528\n        \n        # \u5c1d\u8bd5\u5bfc\u5165OCR\u5e93\n        try:\n            import nagaagent_core.vendors.pytesseract as pytesseract\n            self.ocr_available = True\n            logger.info(\"OCR\u529f\u80fd\u5df2\u542f\u7528\")\n        except ImportError:\n            logger.warning(\"pytesseract\u672a\u5b89\u88c5\uff0cOCR\u529f\u80fd\u4e0d\u53ef\u7528\")\n       \n       # \u5c1d\u8bd5\u52a0\u8f7d\u5404\u4e2a\u5de5\u5177 \n        # \u5c1d\u8bd5\u5bfc\u5165\u56fe\u50cf\u5339\u914d\u5e93\n        try:\n            import nagaagent_core.vendors.cv2 as cv2\n            import numpy as np\n            self.cv2 = cv2\n            self.np = np\n            self.image_matching_available = True\n            logger.info(\"\u56fe\u50cf\u5339\u914d\u529f\u80fd\u5df2\u542f\u7528\")\n        except ImportError:\n            logger.warning(\"opencv-python\u672a\u5b89\u88c5\uff0c\u56fe\u50cf\u5339\u914d\u529f\u80fd\u4e0d\u53ef\u7528\")\n        \n        # \u5c1d\u8bd5\u5bfc\u5165AI\u5750\u6807\u5b9a\u4f4d\u5e93\n        try:\n            from langchain_openai import ChatOpenAI\n            self.ai_coordinate_available = True\n            logger.info(\"AI\u5750\u6807\u5b9a\u4f4d\u529f\u80fd\u5df2\u542f\u7528\")\n        except ImportError:\n            logger.warning(\"langchain-openai\u672a\u5b89\u88c5\uff0cAI\u5750\u6807\u5b9a\u4f4d\u529f\u80fd\u4e0d\u53ef\u7528\")\n    \n    async def analyze_screenshot(self, screenshot: bytes) -> Dict&#91;str, Any]:  #\u5206\u6790\u5c4f\u5e55\u622a\u56fe\uff08\u4e3b\u51fd\u6570\uff09\n        \"\"\"\u5206\u6790\u5c4f\u5e55\u622a\u56fe\"\"\"\n        try:\n            # \u52a0\u8f7d\u56fe\u50cf\n            image = Image.open(io.BytesIO(screenshot))\n            \n            analysis_result = {\n                \"image_size\": image.size,\n                \"mode\": image.mode,\n                \"ocr_text\": &#91;],\n                \"elements\": &#91;],\n                \"analysis_time\": None\n            }\n            \n            # OCR\u6587\u672c\u8bc6\u522b\n            if self.ocr_available:\n                ocr_result = await self._extract_text_from_image(image)\n                analysis_result&#91;\"ocr_text\"] = ocr_result\n            \n            # \u5143\u7d20\u68c0\u6d4b\n            elements = await self._detect_elements(image)\n            analysis_result&#91;\"elements\"] = elements\n            \n            logger.info(f\"\u5c4f\u5e55\u5206\u6790\u5b8c\u6210: \u8bc6\u522b\u5230 {len(analysis_result&#91;'ocr_text'])} \u4e2a\u6587\u672c, {len(elements)} \u4e2a\u5143\u7d20\")\n            return analysis_result\n            \n        except Exception as e:\n            logger.error(f\"\u5c4f\u5e55\u5206\u6790\u5931\u8d25: {e}\")\n            return {\n                \"error\": str(e),\n                \"image_size\": None,\n                \"ocr_text\": &#91;],\n                \"elements\": &#91;]\n            }\n    \n    async def _extract_text_from_image(self, image: Image.Image) -> List&#91;Dict&#91;str, Any]]:   #\u4ece\u56fe\u7247\u63d0\u53d6\u6587\u5b57\n        \"\"\"\u4ece\u56fe\u50cf\u4e2d\u63d0\u53d6\u6587\u672c\"\"\"\n        if not self.ocr_available:\n            return &#91;]\n        \n        try:\n            import nagaagent_core.vendors.pytesseract as pytesseract\n            \n            # \u4f7f\u7528OCR\u63d0\u53d6\u6587\u672c\u548c\u4f4d\u7f6e\u4fe1\u606f\n            data = pytesseract.image_to_data(image, output_type=pytesseract.Output.DICT)\n            \n            text_elements = &#91;]\n            for i in range(len(data&#91;'text'])):\n                text = data&#91;'text']&#91;i].strip()\n                if text:  # \u53ea\u5904\u7406\u975e\u7a7a\u6587\u672c\n                    text_elements.append({\n                        \"text\": text,\n                        \"confidence\": data&#91;'conf']&#91;i],\n                        \"bbox\": {\n                            \"x\": data&#91;'left']&#91;i],\n                            \"y\": data&#91;'top']&#91;i],\n                            \"width\": data&#91;'width']&#91;i],\n                            \"height\": data&#91;'height']&#91;i]\n                        }\n                    })\n            \n            return text_elements\n            \n        except Exception as e:\n            logger.error(f\"OCR\u6587\u672c\u63d0\u53d6\u5931\u8d25: {e}\")\n            return &#91;]\n\n    async def _detect_elements(self, image: Image.Image) -> List&#91;Dict&#91;str, Any]]:   #\u68c0\u6d4b\u56fe\u50cf\u4e2d\u7684\u5143\u7d20\n        \"\"\"\u68c0\u6d4b\u56fe\u50cf\u4e2d\u7684\u5143\u7d20\"\"\"\n        if not self.image_matching_available:\n            return &#91;]\n        \n        try:\n            # \u8f6c\u6362\u4e3aOpenCV\u683c\u5f0f\n            cv_image = self.cv2.cvtColor(self.np.array(image), self.cv2.COLOR_RGB2BGR)\n            \n            # \u68c0\u6d4b\u8fb9\u7f18\n            gray = self.cv2.cvtColor(cv_image, self.cv2.COLOR_BGR2GRAY)\n            edges = self.cv2.Canny(gray, 50, 150)\n            \n            # \u67e5\u627e\u8f6e\u5ed3\n            contours, _ = self.cv2.findContours(edges, self.cv2.RETR_EXTERNAL, self.cv2.CHAIN_APPROX_SIMPLE)\n            \n            elements = &#91;]\n            for contour in contours:\n                # \u8ba1\u7b97\u8fb9\u754c\u6846\n                x, y, w, h = self.cv2.boundingRect(contour)\n                \n                # \u8fc7\u6ee4\u592a\u5c0f\u7684\u5143\u7d20\n                if w > 10 and h > 10:\n                    elements.append({\n                        \"type\": \"contour\",\n                        \"bbox\": {\"x\": x, \"y\": y, \"width\": w, \"height\": h},\n                        \"area\": w * h\n                    })\n            \n            return elements\n            \n        except Exception as e:\n            logger.error(f\"\u5143\u7d20\u68c0\u6d4b\u5931\u8d25: {e}\")\n            return &#91;]\n    \n    async def find_text_element(self, screenshot: bytes, target_text: str) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\u67e5\u627e\u5305\u542b\u6307\u5b9a\u6587\u672c\u7684\u5143\u7d20\u4f4d\u7f6e\"\"\"\n        try:\n            analysis = await self.analyze_screenshot(screenshot)\n            \n            for text_element in analysis.get(\"ocr_text\", &#91;]):\n                if target_text.lower() in text_element&#91;\"text\"].lower():\n                    bbox = text_element&#91;\"bbox\"]\n                    # \u8fd4\u56de\u4e2d\u5fc3\u5750\u6807\n                    center_x = bbox&#91;\"x\"] + bbox&#91;\"width\"] \/\/ 2\n                    center_y = bbox&#91;\"y\"] + bbox&#91;\"height\"] \/\/ 2\n                    return (center_x, center_y)\n            \n            return None\n            \n        except Exception as e:\n            logger.error(f\"\u67e5\u627e\u6587\u672c\u5143\u7d20\u5931\u8d25: {e}\")\n            return None\n    \n    async def find_image_element(self, screenshot: bytes, template_path: str) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\u67e5\u627e\u56fe\u50cf\u6a21\u677f\u5339\u914d\u7684\u5143\u7d20\u4f4d\u7f6e\"\"\"\n        if not self.image_matching_available:\n            return None\n        \n        try:\n            # \u52a0\u8f7d\u6a21\u677f\u56fe\u50cf\n            template = self.cv2.imread(template_path)\n            if template is None:\n                logger.error(f\"\u65e0\u6cd5\u52a0\u8f7d\u6a21\u677f\u56fe\u50cf: {template_path}\")\n                return None\n            \n            # \u52a0\u8f7d\u5c4f\u5e55\u622a\u56fe\n            screen_image = Image.open(io.BytesIO(screenshot))\n            screen_cv = self.cv2.cvtColor(self.np.array(screen_image), self.cv2.COLOR_RGB2BGR)\n            \n            # \u6a21\u677f\u5339\u914d\n            result = self.cv2.matchTemplate(screen_cv, template, self.cv2.TM_CCOEFF_NORMED)\n            min_val, max_val, min_loc, max_loc = self.cv2.minMaxLoc(result)\n            \n            # \u5982\u679c\u5339\u914d\u5ea6\u8db3\u591f\u9ad8\n            if max_val > 0.8:\n                # \u8fd4\u56de\u6a21\u677f\u4e2d\u5fc3\u4f4d\u7f6e\n                center_x = max_loc&#91;0] + template.shape&#91;1] \/\/ 2\n                center_y = max_loc&#91;1] + template.shape&#91;0] \/\/ 2\n                return (center_x, center_y)\n            \n            return None\n            \n        except Exception as e:\n            logger.error(f\"\u56fe\u50cf\u5339\u914d\u5931\u8d25: {e}\")\n            return None\n    \n    async def get_screen_info(self, screenshot: bytes) -> Dict&#91;str, Any]:\n        \"\"\"\u83b7\u53d6\u5c4f\u5e55\u4fe1\u606f\"\"\"\n        try:\n            image = Image.open(io.BytesIO(screenshot))\n            \n            return {\n                \"width\": image.width,\n                \"height\": image.height,\n                \"mode\": image.mode,\n                \"format\": image.format,\n                \"size_bytes\": len(screenshot)\n            }\n            \n        except Exception as e:\n            logger.error(f\"\u83b7\u53d6\u5c4f\u5e55\u4fe1\u606f\u5931\u8d25: {e}\")\n            return {\n                \"error\": str(e),\n                \"width\": 0,\n                \"height\": 0\n            }\n    \n    async def locate_element_with_ai(self, target_description: str, screenshot: bytes, \n                                   screen_width: int = 1920, screen_height: int = 1080) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\n        \u4f7f\u7528AI\u5b9a\u4f4d\u5c4f\u5e55\u5143\u7d20\n        \u652f\u6301\u81ea\u7136\u8bed\u8a00\u63cf\u8ff0\u5b9a\u4f4d\u754c\u9762\u5143\u7d20\n        \"\"\"\n        if not self.ai_coordinate_available:\n            logger.warning(\"AI\u5750\u6807\u5b9a\u4f4d\u529f\u80fd\u4e0d\u53ef\u7528\")\n            return None\n        \n        try:\n            from langchain_openai import ChatOpenAI\n            # \u7edf\u4e00\u4ece\u7cfb\u7edf\u914d\u7f6e\u8bfb\u53d6\u89c6\u89c9LLM\u53c2\u6570\n            from system.config import config\n            cc = getattr(config, 'computer_control', None)\n            model = getattr(cc, 'model', None) or config.api.model\n            base_url = getattr(cc, 'model_url', None) or config.api.base_url\n            api_key = getattr(cc, 'api_key', None) or config.api.api_key\n            \n            # \u521d\u59cb\u5316LLM\n            llm = ChatOpenAI(\n                model=model,\n                base_url=base_url,\n                api_key=api_key,\n                temperature=0\n            )\n            \n            # \u5c06\u622a\u56fe\u8f6c\u6362\u4e3abase64\n            screenshot_b64 = base64.b64encode(screenshot).decode('utf-8')\n            \n            # \u6784\u5efaAI\u5b9a\u4f4d\u63d0\u793a\u8bcd\n            prompt = f\"\"\"\n            \u8bf7\u5206\u6790\u5c4f\u5e55\u622a\u56fe\u5e76\u5b9a\u4f4d\u76ee\u6807\u5143\u7d20: \"{target_description}\"\n            \n            \u8bf7\u6309\u4ee5\u4e0b\u683c\u5f0f\u8f93\u51fa\u5750\u6807\uff1a\n            1. \u5982\u679c\u53ef\u80fd\uff0c\u8f93\u51fa\u8fb9\u754c\u6846 &#91;x1, y1, x2, y2]\uff0c\u5750\u6807\u8303\u56f40-1000\n            2. \u5982\u679c\u8fb9\u754c\u6846\u4e0d\u5408\u9002\uff0c\u8f93\u51fa\u7cbe\u786e\u5750\u6807 x,y\n            3. \u4e0d\u8981\u5305\u542b\u4efb\u4f55\u89e3\u91ca\u6587\u5b57\uff0c\u53ea\u8f93\u51fa\u5750\u6807\n            \n            \u5c4f\u5e55\u5c3a\u5bf8: {screen_width}x{screen_height}\n            \"\"\"\n            \n            # \u8c03\u7528AI\u6a21\u578b\u8fdb\u884c\u5750\u6807\u5b9a\u4f4d\n            response = llm.invoke(&#91;\n                {\n                    \"role\": \"user\", \n                    \"content\": &#91;\n                        {\"type\": \"text\", \"text\": prompt},\n                        {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image\/png;base64,{screenshot_b64}\"}}\n                    ]\n                }\n            ])\n            \n            # \u89e3\u6790AI\u8fd4\u56de\u7684\u5750\u6807\n            coordinates = self._parse_ai_coordinates(response.content, screen_width, screen_height)\n            \n            if coordinates:\n                logger.info(f\"AI\u5b9a\u4f4d\u6210\u529f: {target_description} -> {coordinates}\")\n                return coordinates\n            else:\n                logger.warning(f\"AI\u5b9a\u4f4d\u5931\u8d25: {target_description}\")\n                return None\n                \n        except Exception as e:\n            logger.error(f\"AI\u5750\u6807\u5b9a\u4f4d\u5931\u8d25: {e}\")\n            return None\n    \n    def _parse_ai_coordinates(self, response: str, screen_width: int, screen_height: int) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\n        \u89e3\u6790AI\u8fd4\u56de\u7684\u5750\u6807\n        \u652f\u6301\u8fb9\u754c\u6846\u548c\u7cbe\u786e\u5750\u6807\u4e24\u79cd\u683c\u5f0f\n        \"\"\"\n        try:\n            # \u6e05\u7406\u54cd\u5e94\u6587\u672c\n            response = response.strip()\n            \n            # \u5c1d\u8bd5\u89e3\u6790\u8fb9\u754c\u6846\u683c\u5f0f &#91;x1, y1, x2, y2]\n            bbox_match = re.search(r'\\&#91;(&#91;0-9.,\\s]+)\\]', response)\n            if bbox_match:\n                coords_str = bbox_match.group(1)\n                numbers = re.findall(r'-?\\d+\\.?\\d*', coords_str)\n                if len(numbers) >= 4:\n                    x1, y1, x2, y2 = &#91;float(n) for n in numbers&#91;:4]]\n                    # \u68c0\u67e5\u662f\u5426\u57280-1000\u8303\u56f4\u5185\uff08\u6807\u51c6\u5316\u5750\u6807\uff09\n                    if max(x1, y1, x2, y2) &lt;= 1000:\n                        # \u8ba1\u7b97\u4e2d\u5fc3\u70b9\u5e76\u8f6c\u6362\u4e3a\u5b9e\u9645\u50cf\u7d20\u5750\u6807\n                        center_x = (x1 + x2) \/ 2.0\n                        center_y = (y1 + y2) \/ 2.0\n                        pixel_x = int(round(center_x \/ 1000.0 * screen_width))\n                        pixel_y = int(round(center_y \/ 1000.0 * screen_height))\n                        return (pixel_x, pixel_y)\n                    else:\n                        # \u76f4\u63a5\u4f7f\u7528\u50cf\u7d20\u5750\u6807\n                        center_x = (x1 + x2) \/ 2\n                        center_y = (y1 + y2) \/ 2\n                        return (int(center_x), int(center_y))\n            \n            # \u5c1d\u8bd5\u89e3\u6790\u7cbe\u786e\u5750\u6807\u683c\u5f0f x,y\n            coord_match = re.search(r'(\\d+\\.?\\d*)\\s*,\\s*(\\d+\\.?\\d*)', response)\n            if coord_match:\n                x, y = float(coord_match.group(1)), float(coord_match.group(2))\n                # \u68c0\u67e5\u662f\u5426\u57280-1000\u8303\u56f4\u5185\uff08\u6807\u51c6\u5316\u5750\u6807\uff09\n                if x &lt;= 1000 and y &lt;= 1000:\n                    pixel_x = int(round(x \/ 1000.0 * screen_width))\n                    pixel_y = int(round(y \/ 1000.0 * screen_height))\n                    return (pixel_x, pixel_y)\n                else:\n                    # \u76f4\u63a5\u4f7f\u7528\u50cf\u7d20\u5750\u6807\n                    return (int(x), int(y))\n            \n            # \u5c1d\u8bd5\u89e3\u6790\u6570\u5b57\u5217\u8868\n            numbers = &#91;int(float(x)) for x in re.findall(r'-?\\d+\\.?\\d*', response)]\n            if len(numbers) >= 2:\n                x, y = numbers&#91;0], numbers&#91;1]\n                if x &lt;= 1000 and y &lt;= 1000:\n                    pixel_x = int(round(x \/ 1000.0 * screen_width))\n                    pixel_y = int(round(y \/ 1000.0 * screen_height))\n                    return (pixel_x, pixel_y)\n                else:\n                    return (x, y)\n            \n            return None\n            \n        except Exception as e:\n            logger.error(f\"\u5750\u6807\u89e3\u6790\u5931\u8d25: {e}\")\n            return None\n    \n    async def locate_element(self, target: str, screenshot: bytes) -> Optional&#91;Tuple&#91;int, int]]:\n        \"\"\"\n        \u667a\u80fd\u5143\u7d20\u5b9a\u4f4d\uff0c\u4f18\u5148\u4f7f\u7528AI\u5b9a\u4f4d\uff0c\u56de\u9000\u5230\u4f20\u7edf\u65b9\u6cd5\n        \u591a\u5c42\u6b21\u5b9a\u4f4d\u7b56\u7565\n        \"\"\"\n        try:\n            # \u9996\u5148\u5c1d\u8bd5AI\u5b9a\u4f4d\n            if self.ai_coordinate_available:\n                ai_result = await self.locate_element_with_ai(target, screenshot)\n                if ai_result:\n                    return ai_result\n            \n            # \u56de\u9000\u5230\u6587\u672c\u5b9a\u4f4d\n            if self.ocr_available:\n                text_result = await self.find_text_element(screenshot, target)\n                if text_result:\n                    return text_result\n            \n            # \u56de\u9000\u5230\u56fe\u50cf\u5339\u914d\uff08\u5982\u679ctarget\u662f\u56fe\u50cf\u8def\u5f84\uff09\n            if self.image_matching_available and target.endswith(('.png', '.jpg', '.jpeg')):\n                image_result = await self.find_image_element(screenshot, target)\n                if image_result:\n                    return image_result\n            \n            logger.warning(f\"\u65e0\u6cd5\u5b9a\u4f4d\u5143\u7d20: {target}\")\n            return None\n            \n        except Exception as e:\n            logger.error(f\"\u5143\u7d20\u5b9a\u4f4d\u5931\u8d25: {e}\")\n            return None\n    \n    def is_available(self) -> Dict&#91;str, Any]:\n        \"\"\"\u68c0\u67e5\u89c6\u89c9\u5206\u6790\u5668\u53ef\u7528\u6027\"\"\"\n        return {\n            \"ocr_available\": self.ocr_available,\n            \"image_matching_available\": self.image_matching_available,\n            \"ai_coordinate_available\": self.ai_coordinate_available,\n            \"ready\": self.ocr_available or self.image_matching_available or self.ai_coordinate_available\n        }\n    \n# ==========\u4ee3\u7801\u91cc\u903b\u8f91\u5224\u65ad======\n#\u8fd9\u4e2a\u4ee3\u7801\u91cc\u9762\u7684\u529f\u80fd\u5176\u5b9e\u90fd\u662f\u5341\u5206\u76f8\u4f3c\u7684\uff0c\u5224\u65ad\u903b\u8f91\u4e5f\u5341\u5206\u76f8\u540c\uff0c\u6240\u4ee5\u6211\u4eec\u91cd\u70b9\u8bb2\u4e00\u4e0b\u4e00\u4e9b\u8fd9\u4e2a\u4ee3\u7801\u91cc\u9762\uff0c\u5904\u5904\u53ef\u89c1\u7684\uff0c\u91cd\u590d\u51fa\u73b0\u7684\u4e1c\u897f\u5c31\u884c\u4e86\u3002\n'''\n1. \u9519\u8bef\u5904\u7406\u903b\u8f91\uff08\u5904\u5904\u53ef\u89c1\uff09\ntry:\n    # \u5c1d\u8bd5\u505a\u67d0\u4ef6\u4e8b\n    do_something()\nexcept Exception as e:  # \u5982\u679c\u51fa\u9519\u4e86\n    logger.error(f\"\u51fa\u9519\u4e86: {e}\")  # \u8bb0\u5f55\u9519\u8bef\n    return &#91;]  # \u8fd4\u56de\u7a7a\u7ed3\u679c\uff0c\u4e0d\u8ba9\u7a0b\u5e8f\u5d29\u6e83\n    \n\u4e3a\u4ec0\u4e48\u8fd9\u6837\u8bbe\u8ba1\uff1f\n\u4e0d\u80fd\u8ba9\u4e00\u4e2a\u529f\u80fd\u51fa\u9519\u5bfc\u81f4\u6574\u4e2a\u7a0b\u5e8f\u5d29\u6e83\n\u6bd4\u5982OCR\u529f\u80fd\u574f\u4e86\uff0c\u56fe\u50cf\u5339\u914d\u8fd8\u80fd\u7528\n\n2. \u529f\u80fd\u53ef\u7528\u6027\u68c0\u67e5\ndef is_available(self) -> Dict&#91;str, Any]:\n    return {\n        \"ocr_available\": self.ocr_available,\n        \"image_matching_available\": self.image_matching_available,\n        \"ai_coordinate_available\": self.ai_coordinate_available,\n        \"ready\": self.ocr_available or self.image_matching_available or self.ai_coordinate_available\n    }\n\u8fd9\u4e2a\u51fd\u6570\u544a\u8bc9\u7528\u6237\uff1a\u6211\u73b0\u5728\u6709\u54ea\u4e9b\u5de5\u5177\u53ef\u7528\n\n3. AI\u5750\u6807\u89e3\u6790\u7684\u667a\u80fd\u5224\u65ad\ndef _parse_ai_coordinates(self, response: str, screen_width: int, screen_height: int):\nAI\u53ef\u80fd\u8fd4\u56de\u4e24\u79cd\u683c\u5f0f\uff1a\n  &#91;x1, y1, x2, y2] - \u4e00\u4e2a\u6846\u7684\u56db\u4e2a\u89d2\n  x,y - \u4e00\u4e2a\u70b9\u7684\u5750\u6807\n\u7a0b\u5e8f\u8981\u80fd\u8bc6\u522b\u51fa\u662f\u54ea\u79cd\u683c\u5f0f\u5e76\u6b63\u786e\u5904\u7406\n'''\n#\u8fd9\u4e2a\u6587\u4ef6\u7684\u6574\u4f53\u903b\u8f91\u6d41\u7a0b\u56fe\n'''\n\u5f00\u59cb\u4f7f\u7528\u89c6\u89c9\u5206\u6790\u5668\n    \u2193\n\u521d\u59cb\u5316\u5de5\u5177\u7bb1\n    \u251c\u2500 \u68c0\u67e5OCR\u5de5\u5177 \u2713\n    \u251c\u2500 \u68c0\u67e5\u56fe\u50cf\u5339\u914d\u5de5\u5177 \u2713\n    \u2514\u2500 \u68c0\u67e5AI\u5de5\u5177 \u2713\n    \u2193\n\u7528\u6237\u60f3\u627e\u4e1c\u897f\u65f6\uff1a\n    \u2193\n\u4f18\u5148\u7528AI\u5b9a\u4f4d\uff08\u5982\u679c\u53ef\u7528\uff09\n    \u2193\n\u5982\u679cAI\u627e\u4e0d\u5230\u6216\u4e0d\u53ef\u7528 \u2192 \u7528\u6587\u5b57\u8bc6\u522b\n    \u2193\n\u5982\u679c\u6587\u5b57\u627e\u4e0d\u5230 \u2192 \u7528\u56fe\u50cf\u5339\u914d\n    \u2193\n\u8fd4\u56de\u627e\u5230\u7684\u4f4d\u7f6e \u6216 \u8fd4\u56de\u201c\u627e\u4e0d\u5230\u201d\n'''<\/code><\/pre>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>computer_control_agent action_executor.py computer_use_ [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[27,55],"tags":[],"class_list":["post-1277","post","type-post","status-publish","format-standard","hentry","category-open-source-analysis","category-technology"],"_links":{"self":[{"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/posts\/1277","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/comments?post=1277"}],"version-history":[{"count":2,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/posts\/1277\/revisions"}],"predecessor-version":[{"id":1280,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/posts\/1277\/revisions\/1280"}],"wp:attachment":[{"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/media?parent=1277"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/categories?post=1277"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.preluna.xyz\/index.php\/wp-json\/wp\/v2\/tags?post=1277"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}