initial commit

client/cloud_run_client.py

@@ -0,0 +1,80 @@
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    from googleapiclient import discovery
+    from google.oauth2 import service_account
+except ImportError:
+    discovery = None  # type: ignore
+    service_account = None  # type: ignore
+
+
+class CloudRunClient:
+    SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
+
+    def __init__(self, project_id: str, region: str = "us-central1",
+                 credentials_file: Optional[str] = None) -> None:
+        self.project_id = project_id
+        self.region     = region
+        if discovery is None:
+            raise ImportError("google-api-python-client is not installed")
+        if credentials_file and service_account:
+            creds = service_account.Credentials.from_service_account_file(
+                credentials_file, scopes=self.SCOPES
+            )
+        else:
+            import google.auth
+            creds, _ = google.auth.default(scopes=self.SCOPES)
+        self._svc = discovery.build("run", "v1", credentials=creds)
+        self._parent = f"namespaces/{project_id}"
+        logger.debug("Cloud Run client for %s/%s", project_id, region)
+
+    def list_services(self) -> list[dict[str, Any]]:
+        response = (
+            self._svc.namespaces()
+            .services()
+            .list(parent=self._parent)
+            .execute()
+        )
+        return [
+            {
+                "name":   svc["metadata"]["name"],
+                "url":    svc.get("status", {}).get("url", ""),
+                "ready":  any(
+                    c.get("type") == "Ready" and c.get("status") == "True"
+                    for c in svc.get("status", {}).get("conditions", [])
+                ),
+            }
+            for svc in response.get("items", [])
+        ]
+
+    def get_service(self, service_name: str) -> dict[str, Any]:
+        name = f"{self._parent}/services/{service_name}"
+        svc  = self._svc.namespaces().services().get(name=name).execute()
+        return {
+            "name":        svc["metadata"]["name"],
+            "url":         svc.get("status", {}).get("url"),
+            "image":       (svc.get("spec", {})
+                             .get("template", {})
+                             .get("spec", {})
+                             .get("containers", [{}])[0]
+                             .get("image")),
+            "annotations": svc.get("metadata", {}).get("annotations", {}),
+        }
+
+    def update_traffic(self, service_name: str,
+                       traffic: list[dict[str, Any]]) -> None:
+        name = f"{self._parent}/services/{service_name}"
+        svc  = self._svc.namespaces().services().get(name=name).execute()
+        svc["spec"]["traffic"] = traffic
+        self._svc.namespaces().services().replaceService(
+            name=name, body=svc
+        ).execute()
+        logger.info("Updated traffic for service %s", service_name)
+
+    def delete_service(self, service_name: str) -> None:
+        name = f"{self._parent}/services/{service_name}"
+        self._svc.namespaces().services().delete(name=name).execute()
+        logger.info("Deleted Cloud Run service %s", service_name)

client/cloud_sql_client.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Any, Iterator, Optional
+import contextlib
+
+logger = logging.getLogger(__name__)
+
+try:
+    import sqlalchemy
+    from sqlalchemy import text
+except ImportError:
+    sqlalchemy = None  # type: ignore
+
+
+class CloudSQLClient:
+    def __init__(self, connection_name: str, db_name: str,
+                 user: str, password: str,
+                 use_public_ip: bool = False) -> None:
+        self.connection_name = connection_name
+        self.db_name         = db_name
+        self.user            = user
+        self._password       = password
+        self.use_public_ip   = use_public_ip
+        self._engine = None
+        logger.debug("CloudSQL client for %s/%s", connection_name, db_name)
+
+    def _get_engine(self):
+        if self._engine is not None:
+            return self._engine
+        if sqlalchemy is None:
+            raise ImportError("sqlalchemy is not installed")
+        try:
+            from google.cloud.sql.connector import Connector
+            connector = Connector()
+
+            def getconn():
+                return connector.connect(
+                    self.connection_name, "pg8000",
+                    user=self.user, password=self._password,
+                    db=self.db_name, ip_type="PUBLIC" if self.use_public_ip else "PRIVATE",
+                )
+
+            self._engine = sqlalchemy.create_engine(
+                "postgresql+pg8000://", creator=getconn,
+                pool_size=5, max_overflow=2, pool_timeout=30,
+            )
+        except ImportError:
+            url = (f"postgresql+pg8000://{self.user}:{self._password}"
+                   f"@localhost/{self.db_name}")
+            self._engine = sqlalchemy.create_engine(url)
+        return self._engine
+
+    @contextlib.contextmanager
+    def connection(self):
+        engine = self._get_engine()
+        with engine.connect() as conn:
+            yield conn
+
+    def execute(self, sql: str, params: Optional[dict] = None) -> int:
+        with self.connection() as conn:
+            result = conn.execute(text(sql), params or {})
+            conn.commit()
+            return result.rowcount
+
+    def fetch_all(self, sql: str,
+                  params: Optional[dict] = None) -> list[dict[str, Any]]:
+        with self.connection() as conn:
+            rows = conn.execute(text(sql), params or {}).fetchall()
+            return [dict(row._mapping) for row in rows]
+
+    def fetch_one(self, sql: str,
+                  params: Optional[dict] = None) -> Optional[dict[str, Any]]:
+        with self.connection() as conn:
+            row = conn.execute(text(sql), params or {}).fetchone()
+            return dict(row._mapping) if row else None
+
+    def table_exists(self, schema: str, table: str) -> bool:
+        result = self.fetch_one(
+            "SELECT 1 FROM information_schema.tables "
+            "WHERE table_schema = :schema AND table_name = :table",
+            {"schema": schema, "table": table},
+        )
+        return result is not None
+
+    def run_migration(self, migration_sql: str) -> None:
+        logger.info("Running migration (%d chars)", len(migration_sql))
+        with self.connection() as conn:
+            for statement in migration_sql.split(";"):
+                stmt = statement.strip()
+                if stmt:
+                    conn.execute(text(stmt))
+            conn.commit()
+        logger.info("Migration complete")

client/compute_client.py

@@ -0,0 +1,99 @@
+import logging
+import time
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    from google.cloud import compute_v1
+    from google.api_core.exceptions import NotFound
+except ImportError:
+    compute_v1 = None  # type: ignore
+    NotFound = Exception
+
+
+class ComputeClient:
+    def __init__(self, project_id: str, zone: str,
+                 credentials_file: Optional[str] = None) -> None:
+        self.project_id = project_id
+        self.zone       = zone
+        if compute_v1 is None:
+            raise ImportError("google-cloud-compute is not installed")
+        self._instances = compute_v1.InstancesClient()
+        self._operations = compute_v1.ZoneOperationsClient()
+        logger.debug("Compute client for %s/%s", project_id, zone)
+
+    def _wait_for_operation(self, operation_name: str,
+                            timeout: int = 300) -> None:
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            op = self._operations.get(project=self.project_id,
+                                      zone=self.zone,
+                                      operation=operation_name)
+            if op.status == compute_v1.Operation.Status.DONE:
+                if op.error:
+                    raise RuntimeError(f"Operation failed: {op.error}")
+                return
+            time.sleep(5)
+        raise TimeoutError(f"Operation {operation_name} timed out")
+
+    def list_instances(self, filter_expr: str = "") -> list[dict[str, Any]]:
+        request = compute_v1.ListInstancesRequest(
+            project=self.project_id, zone=self.zone, filter=filter_expr
+        )
+        instances = []
+        for inst in self._instances.list(request=request):
+            instances.append({
+                "name":   inst.name,
+                "status": inst.status,
+                "machine_type": inst.machine_type.split("/")[-1],
+                "zone":   self.zone,
+            })
+        return instances
+
+    def get_instance(self, name: str) -> dict[str, Any]:
+        inst = self._instances.get(project=self.project_id,
+                                   zone=self.zone, instance=name)
+        return {
+            "name":         inst.name,
+            "status":       inst.status,
+            "machine_type": inst.machine_type.split("/")[-1],
+            "network_ips":  [
+                iface.network_i_p
+                for iface in inst.network_interfaces
+            ],
+        }
+
+    def start_instance(self, name: str) -> None:
+        op = self._instances.start(project=self.project_id,
+                                   zone=self.zone, instance=name)
+        self._wait_for_operation(op.name)
+        logger.info("Started instance %s", name)
+
+    def stop_instance(self, name: str) -> None:
+        op = self._instances.stop(project=self.project_id,
+                                  zone=self.zone, instance=name)
+        self._wait_for_operation(op.name)
+        logger.info("Stopped instance %s", name)
+
+    def instance_status(self, name: str) -> str:
+        try:
+            inst = self._instances.get(project=self.project_id,
+                                       zone=self.zone, instance=name)
+            return inst.status
+        except NotFound:
+            return "NOT_FOUND"
+
+    def set_labels(self, name: str, labels: dict[str, str]) -> None:
+        inst = self._instances.get(project=self.project_id,
+                                   zone=self.zone, instance=name)
+        req = compute_v1.SetLabelsInstanceRequest(
+            project=self.project_id, zone=self.zone, instance=name,
+            instances_set_labels_request_resource=compute_v1.InstancesSetLabelsRequest(
+                label_fingerprint=inst.label_fingerprint,
+                labels=labels,
+            ),
+        )
+        op = self._instances.set_labels(request=req)
+        self._wait_for_operation(op.name)
+        logger.info("Labels updated for instance %s", name)

client/logger_setup.py

@@ -0,0 +1,51 @@
+import logging
+import sys
+from typing import Optional
+
+
+def setup(
+    name: str = "gcp-tool",
+    level: str = "INFO",
+    log_file: Optional[str] = None,
+    fmt: str = "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+) -> logging.Logger:
+    logger = logging.getLogger(name)
+    logger.setLevel(getattr(logging, level.upper(), logging.INFO))
+    logger.handlers.clear()
+
+    formatter = logging.Formatter(fmt, datefmt="%Y-%m-%dT%H:%M:%S")
+
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    logger.propagate = False
+    return logger
+
+
+class StructuredLogger:
+    def __init__(self, name: str) -> None:
+        self._log = logging.getLogger(name)
+
+    def _fmt(self, msg: str, **kw) -> str:
+        if not kw:
+            return msg
+        pairs = " ".join(f"{k}={v!r}" for k, v in kw.items())
+        return f"{msg} {pairs}"
+
+    def info(self, msg: str, **kw) -> None:
+        self._log.info(self._fmt(msg, **kw))
+
+    def warning(self, msg: str, **kw) -> None:
+        self._log.warning(self._fmt(msg, **kw))
+
+    def error(self, msg: str, **kw) -> None:
+        self._log.error(self._fmt(msg, **kw))
+
+    def debug(self, msg: str, **kw) -> None:
+        self._log.debug(self._fmt(msg, **kw))

config/pubsub_client.py

@@ -0,0 +1,104 @@
+import json
+import logging
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+try:
+    from google.cloud import pubsub_v1
+    from google.api_core.exceptions import AlreadyExists, NotFound
+except ImportError:
+    pubsub_v1 = None  # type: ignore
+    AlreadyExists = Exception
+    NotFound = Exception
+
+
+class PubSubClient:
+    def __init__(self, project_id: str,
+                 credentials_file: Optional[str] = None) -> None:
+        self.project_id = project_id
+        if pubsub_v1 is None:
+            raise ImportError("google-cloud-pubsub is not installed")
+        kw: dict = {}
+        if credentials_file:
+            from google.oauth2 import service_account
+            creds = service_account.Credentials.from_service_account_file(
+                credentials_file
+            )
+            kw["credentials"] = creds
+        self._publisher  = pubsub_v1.PublisherClient(**kw)
+        self._subscriber = pubsub_v1.SubscriberClient(**kw)
+        logger.debug("Pub/Sub client initialised for project %s", project_id)
+
+    def _topic_path(self, topic: str) -> str:
+        return self._publisher.topic_path(self.project_id, topic)
+
+    def _sub_path(self, subscription: str) -> str:
+        return self._subscriber.subscription_path(self.project_id, subscription)
+
+    # ── topics ─────────────────────────────────────────────────────────
+
+    def create_topic(self, topic: str) -> None:
+        try:
+            self._publisher.create_topic(request={"name": self._topic_path(topic)})
+            logger.info("Created topic %s", topic)
+        except AlreadyExists:
+            logger.debug("Topic %s already exists", topic)
+
+    def delete_topic(self, topic: str) -> None:
+        self._publisher.delete_topic(request={"topic": self._topic_path(topic)})
+        logger.info("Deleted topic %s", topic)
+
+    def list_topics(self) -> list[str]:
+        path = f"projects/{self.project_id}"
+        return [t.name.split("/")[-1]
+                for t in self._publisher.list_topics(request={"project": path})]
+
+    # ── publish ─────────────────────────────────────────────────────────
+
+    def publish(self, topic: str, data: Any,
+                attributes: Optional[dict[str, str]] = None) -> str:
+        encoded = json.dumps(data).encode() if not isinstance(data, bytes) else data
+        future = self._publisher.publish(
+            self._topic_path(topic),
+            data=encoded,
+            **(attributes or {}),
+        )
+        message_id = future.result()
+        logger.debug("Published message %s to %s", message_id, topic)
+        return message_id
+
+    # ── subscribe ────────────────────────────────────────────────────────
+
+    def create_subscription(self, topic: str, subscription: str,
+                             ack_deadline: int = 60) -> None:
+        try:
+            self._subscriber.create_subscription(request={
+                "name": self._sub_path(subscription),
+                "topic": self._topic_path(topic),
+                "ack_deadline_seconds": ack_deadline,
+            })
+            logger.info("Created subscription %s → %s", subscription, topic)
+        except AlreadyExists:
+            logger.debug("Subscription %s already exists", subscription)
+
+    def pull(self, subscription: str, max_messages: int = 10) -> list[dict]:
+        response = self._subscriber.pull(request={
+            "subscription": self._sub_path(subscription),
+            "max_messages": max_messages,
+        })
+        messages = []
+        ack_ids  = []
+        for msg in response.received_messages:
+            messages.append({
+                "message_id": msg.message.message_id,
+                "data": json.loads(msg.message.data.decode()),
+                "attributes": dict(msg.message.attributes),
+            })
+            ack_ids.append(msg.ack_id)
+        if ack_ids:
+            self._subscriber.acknowledge(request={
+                "subscription": self._sub_path(subscription),
+                "ack_ids": ack_ids,
+            })
+        return messages

config/schema.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Optional
+
+
+@dataclass
+class TableSchema:
+    name: str
+    fields: list[FieldSchema] = field(default_factory=list)
+
+    def add_field(self, name: str, field_type: str,
+                  mode: str = "NULLABLE", description: str = "") -> None:
+        self.fields.append(FieldSchema(name=name, field_type=field_type,
+                                       mode=mode, description=description))
+
+    def to_bq_schema(self) -> list[dict]:
+        return [f.to_dict() for f in self.fields]
+
+    def field_names(self) -> list[str]:
+        return [f.name for f in self.fields]
+
+
+@dataclass
+class FieldSchema:
+    name: str
+    field_type: str
+    mode: str = "NULLABLE"
+    description: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "name": self.name,
+            "type": self.field_type,
+            "mode": self.mode,
+            "description": self.description,
+        }
+
+
+@dataclass
+class JobResult:
+    job_id: str
+    state: str
+    started_at: Optional[datetime] = None
+    ended_at: Optional[datetime] = None
+    rows_processed: int = 0
+    bytes_processed: int = 0
+    errors: list[str] = field(default_factory=list)
+
+    @property
+    def duration_seconds(self) -> Optional[float]:
+        if self.started_at and self.ended_at:
+            return (self.ended_at - self.started_at).total_seconds()
+        return None
+
+    def succeeded(self) -> bool:
+        return self.state == "DONE" and not self.errors
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "job_id": self.job_id,
+            "state": self.state,
+            "duration_s": self.duration_seconds,
+            "rows_processed": self.rows_processed,
+            "bytes_processed": self.bytes_processed,
+            "errors": self.errors,
+        }

exceptions.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+
+class GCPError(Exception):
+    def __init__(self, message: str, cause: Optional[Exception] = None) -> None:
+        super().__init__(message)
+        self.cause = cause
+
+    def __str__(self) -> str:
+        base = super().__str__()
+        return f"{base} (caused by: {self.cause})" if self.cause else base
+
+
+class AuthenticationError(GCPError):
+    pass
+
+
+class ResourceNotFoundError(GCPError):
+    def __init__(self, resource_type: str, name: str) -> None:
+        super().__init__(f"{resource_type} not found: {name}")
+        self.resource_type = resource_type
+        self.name = name
+
+
+class QuotaExceededError(GCPError):
+    pass
+
+
+class PermissionDeniedError(GCPError):
+    pass
+
+
+class RetryableError(GCPError):
+    def __init__(self, message: str, attempt: int, cause: Optional[Exception] = None) -> None:
+        super().__init__(message, cause)
+        self.attempt = attempt
+
+
+class TimeoutError(GCPError):
+    pass

gcp_secret.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "honeypot-491918",
+  "private_key_id": "a5d4bfaba95b382bb90a602fbade9915c6996a03",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCq3+J412MyO/Tk\nDXzmEckKm/LCBCyNpNx8obsPdlfG2Ssqjm+LQnJQijqj5eTwBl1Yskg2pec9VHMH\nQ7AScKdJvc1hUgHUEr1oUZ79sVb6bekfak8rTDMVO3VJLs/HxY2CIhAb0zdqCX4p\n0MfXs+RWSkn5U+5zWYctItbkqQYkcUq7lnecjiWCYhvJrARx/9ksQ22zFNp0Rv8c\nmLc4p9kod0Pcaet1kdHq72YJbXajbD9lJmltgpDq5dnS6Ok7XZLSq6c9fuvGPI5y\nJ2xHfseGmA94r88WJrUn2JF1inb4F3qSmhX7vEk64rZoAQSabHjBwzn5MePhfgAo\n7B2lOuBnAgMBAAECggEABG5AYxMYAV9zSXkyueHQLV7p0S9mha22rIyiWxOd0bbO\nhbtNF8v7RtdZZJKQzcsnHyZQSl/MOoI2vAN8ofMbLgWmjlV1XVhht5Nrj0x1OD0i\nrwEvWFdmI7DM/ml2Ux8Y55JBNKzUt60oq6Y8PahDzXD+W2FabNHveAjP5adJqibd\nrKWttAJfLyjGJomrUgdcjRQ9k70dnsXk52x8xLEuDYWDufKbmafoKFwfMpI+G7ML\n5fFqWZ8MTAu15HI3JueaMBHs4sN3NOHzzVI97RSakDjr4psjeVKiGmlmtgn5g9wr\n41rjSKzvwatZeEMPVJ853kcmxhVnoCqPt+lzACgdUQKBgQDgxiEZUr8h9PlWUAqL\nY0FArINJVoiHQIY17aQ37Rx1osuNbfkHFTuvfHLiLxazLkZ0caD171LVcHQKc8/w\ndg6SzdAwQjCU/mFiSEpgoUE5oc1hHneAhzIiL+xGy6P6vgdM0QsbxchXeFcFofET\noNJqmkQkAR/Jq7bzB3CkaNeMQwKBgQDCnN/5TRyRbq76AQok103S+/cuG0OCHtYG\njKYht1C5ljTqPBMOhJDqQAt+7YFYNJjY9omGUXwSlqopVaQHkPh24Px2k5uTSddA\nwtBncgliLpYJ6q/opr3TMAnZqX7oNzKqKWNk71D6tR9A85QR9ugd/yuRtyo/77+j\nAPernIirDQKBgHPgcIvXhsWAv1Hp7aWyQ5+N0BjszXG8p/QBURmr2XIjvQBqY2lt\nXtm3NJq6X997ycKaGbrv6Co00P3aeqailhzEkcFvPhBxm87JKd7rAu8tJO4egMj3\n6baKMirFLQJFGQpW8n0c142Qcnxq4heWXyOp59sP4U1cgvKDMWTwz751AoGBALHO\n7vwyLhXaUNhutrvuU1afi7l+g3GRhu649tj7SjAmQvZ4KNks2dKM7XPswC6876Hb\nrs8gCAJb0eCuNI4NMFY9Pa2BM7Wg06CB3jOFh3lDIyawXERuhCU+UOGj8SVtuUar\nRfrkhNJrRcDsS68WLgGqMD6UYW/FNwxRR/htrpjJAoGBAJIQtnCfFSw2sdNMTIbg\nyR+F1plTUualSqKnWwDbR3rMj2iLWqCfbYcvBz0i2vupqzMn6oOolrfD8hP2hJ5x\n6foaVEm4XTdRUXPHFePmi65GY2lzYek2NXLVEhaaZY/vfEFD7Ab3hZhm+CGoLcXK\noiZ1aR6DYGxXHFOzKN3vM9Tr\n-----END PRIVATE KEY-----\n",
+  "client_email": "token04@honeypot-491918.iam.gserviceaccount.com",
+  "client_id": "102923772131939822524",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/token04%40honeypot-491918.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}