1148 lines
34 KiB
Python
1148 lines
34 KiB
Python
import json
|
|
import os
|
|
import time
|
|
|
|
import frappe
|
|
import requests
|
|
import urllib3
|
|
|
|
# Suppress SSL warnings
|
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
SERVERS_FILE = "/home/frappe/frappe-bench/apps/admin_panel/templates/servers.json"
|
|
CERTS_DIR = "/home/frappe/frappe-bench/apps/admin_panel/certs"
|
|
CLIENT_CERT = os.path.join(CERTS_DIR, "client.crt")
|
|
CLIENT_KEY = os.path.join(CERTS_DIR, "client.key")
|
|
|
|
DEFAULT_VOLUMES = [
|
|
{"name": "mysql", "source_path": "/var/lib/mysql", "device_name": "mysql"},
|
|
{"name": "frontend", "source_path": "/home/frappe/frappe-bench/sites/frontend", "device_name": "frontend"},
|
|
]
|
|
|
|
|
|
def _check_admin_permission():
|
|
"""Check if user has admin permission."""
|
|
if frappe.session.user == "Guest":
|
|
frappe.throw("Please login to access this feature", frappe.AuthenticationError)
|
|
if frappe.session.user == "Administrator":
|
|
return
|
|
roles = frappe.get_roles()
|
|
if "System Manager" not in roles and "Administrator" not in roles:
|
|
frappe.throw("You need System Manager or Administrator role", frappe.PermissionError)
|
|
|
|
|
|
def _load_servers():
|
|
"""Load servers config from JSON file."""
|
|
if os.path.exists(SERVERS_FILE):
|
|
with open(SERVERS_FILE, "r") as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
|
|
def _get_server_config(server_name):
|
|
"""Get server URL and client certificate paths."""
|
|
servers = _load_servers()
|
|
server = servers.get(server_name, {})
|
|
url = server.get("url", "")
|
|
|
|
return {
|
|
"url": url,
|
|
"cert": CLIENT_CERT if os.path.exists(CLIENT_CERT) else None,
|
|
"key": CLIENT_KEY if os.path.exists(CLIENT_KEY) else None,
|
|
}
|
|
|
|
|
|
def _incus_api(server_name, method, path, data=None, timeout=30):
|
|
"""Make REST API request to a specific Incus server."""
|
|
config = _get_server_config(server_name)
|
|
|
|
if not config["url"]:
|
|
raise Exception(f"Server '{server_name}' URL not found")
|
|
|
|
if not config["cert"] or not config["key"]:
|
|
raise Exception("Panel certificate not configured. Please generate a certificate in Servers tab.")
|
|
|
|
url = f"{config['url']}{path}"
|
|
|
|
try:
|
|
r = requests.request(
|
|
method,
|
|
url,
|
|
headers={"Content-Type": "application/json"},
|
|
cert=(config["cert"], config["key"]),
|
|
verify=False,
|
|
json=data,
|
|
timeout=timeout,
|
|
)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
except requests.exceptions.ConnectionError:
|
|
raise Exception(f"Cannot connect to server '{server_name}' at {config['url']}")
|
|
except requests.exceptions.Timeout:
|
|
raise Exception(f"Connection to '{server_name}' timed out")
|
|
except requests.exceptions.HTTPError as e:
|
|
raise Exception(f"Server '{server_name}' error: {e.response.status_code}")
|
|
|
|
|
|
def _wait_for_operation(server_name, operation_url, timeout=60):
|
|
"""Wait for an Incus async operation to complete."""
|
|
if not operation_url:
|
|
return
|
|
wait_path = f"{operation_url}/wait"
|
|
try:
|
|
result = _incus_api(server_name, "GET", f"{wait_path}?timeout={timeout}")
|
|
return result
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _extract_ip(state_data):
|
|
"""Extract IPv4 address from Incus state network data."""
|
|
network = state_data.get("metadata", {}).get("network", {})
|
|
for iface_name, iface in network.items():
|
|
if iface_name == "lo":
|
|
continue
|
|
for addr in iface.get("addresses", []):
|
|
if addr.get("family") == "inet" and addr.get("scope") == "global":
|
|
return addr.get("address", "")
|
|
return ""
|
|
|
|
|
|
def _get_cloud_containers():
|
|
"""Get data from Cloud Container doctype if available."""
|
|
try:
|
|
containers = frappe.get_all(
|
|
"Cloud Container",
|
|
fields=["name", "container_name", "image", "owner", "ip_address", "status"],
|
|
)
|
|
return {c.get("container_name", c.get("name")): c for c in containers}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_images(target_server=None):
|
|
"""Get available images from a specific Incus server."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
return {"error": "Server name is required"}
|
|
|
|
try:
|
|
resp = _incus_api(target_server, "GET", "/1.0/images?recursion=1")
|
|
except Exception as e:
|
|
return {"error": str(e)}
|
|
|
|
images = []
|
|
for img in resp.get("metadata", []):
|
|
aliases = img.get("aliases", [])
|
|
props = img.get("properties", {})
|
|
fingerprint = img.get("fingerprint", "")
|
|
|
|
alias = aliases[0]["name"] if aliases else ""
|
|
description = props.get("description", "") or alias or fingerprint[:12]
|
|
|
|
images.append({
|
|
"alias": alias,
|
|
"description": description,
|
|
"os": props.get("os", ""),
|
|
"architecture": img.get("architecture", ""),
|
|
"size": img.get("size", 0),
|
|
"fingerprint": fingerprint,
|
|
"created_at": img.get("created_at", ""),
|
|
})
|
|
|
|
return images
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_all_containers(target_server=None):
|
|
"""Get list of all containers from a specific Incus server."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
return {"error": "Server name is required"}
|
|
|
|
try:
|
|
resp = _incus_api(target_server, "GET", "/1.0/instances?recursion=1")
|
|
except Exception as e:
|
|
return {"error": str(e)}
|
|
|
|
cloud_data = _get_cloud_containers()
|
|
|
|
containers = []
|
|
for inst in resp.get("metadata", []):
|
|
name = inst.get("name", "")
|
|
status = inst.get("status", "Unknown")
|
|
config = inst.get("config", {})
|
|
created_at = inst.get("created_at", "")
|
|
architecture = inst.get("architecture", "")
|
|
container_type = inst.get("type", "container")
|
|
|
|
# Get IP from state if running
|
|
ip_address = ""
|
|
state = inst.get("state", {})
|
|
if state and status == "Running":
|
|
network = state.get("network", {})
|
|
for iface_name, iface in network.items():
|
|
if iface_name == "lo":
|
|
continue
|
|
for addr in iface.get("addresses", []):
|
|
if addr.get("family") == "inet" and addr.get("scope") == "global":
|
|
ip_address = addr.get("address", "")
|
|
break
|
|
if ip_address:
|
|
break
|
|
|
|
image = config.get("image.description", config.get("image.os", ""))
|
|
cloud = cloud_data.get(name, {})
|
|
|
|
containers.append({
|
|
"name": name,
|
|
"container_name": name,
|
|
"status": status,
|
|
"ip_address": ip_address or cloud.get("ip_address", ""),
|
|
"image": image or cloud.get("image", ""),
|
|
"owner": cloud.get("owner", ""),
|
|
"created": created_at[:10] if created_at else "",
|
|
"architecture": architecture,
|
|
"type": container_type,
|
|
"server": target_server,
|
|
})
|
|
|
|
return containers
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_container_detail(container_name, target_server=None):
|
|
"""Get detailed container info including resource usage."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
return {"error": "Server name is required"}
|
|
|
|
try:
|
|
detail = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
except Exception as e:
|
|
return {"error": str(e), "name": container_name, "status": "Error"}
|
|
|
|
meta = detail.get("metadata", {})
|
|
config = meta.get("config", {})
|
|
status = meta.get("status", "Unknown")
|
|
created_at = meta.get("created_at", "")
|
|
|
|
result = {
|
|
"name": container_name,
|
|
"container_name": container_name,
|
|
"status": status,
|
|
"ip_address": "",
|
|
"image": config.get("image.description", config.get("image.os", "")),
|
|
"owner": "",
|
|
"created": created_at[:10] if created_at else "",
|
|
"architecture": meta.get("architecture", ""),
|
|
"type": meta.get("type", "container"),
|
|
"profiles": meta.get("profiles", []),
|
|
"cpu_usage": 0,
|
|
"ram_usage": 0,
|
|
"disk_usage": 0,
|
|
"memory_usage_bytes": 0,
|
|
"memory_total_bytes": 0,
|
|
"disk_usage_bytes": 0,
|
|
"disk_total_bytes": 0,
|
|
"network_rx": 0,
|
|
"network_tx": 0,
|
|
"connections": 0,
|
|
"uptime": "-",
|
|
"server": target_server,
|
|
}
|
|
|
|
# Enrich with Cloud Container data
|
|
cloud_data = _get_cloud_containers()
|
|
cloud = cloud_data.get(container_name, {})
|
|
if cloud:
|
|
result["owner"] = cloud.get("owner", "")
|
|
if not result["image"]:
|
|
result["image"] = cloud.get("image", "")
|
|
|
|
# Get state if running
|
|
if status == "Running":
|
|
try:
|
|
state = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}/state")
|
|
state_meta = state.get("metadata", {})
|
|
|
|
result["ip_address"] = _extract_ip(state)
|
|
if not result["ip_address"] and cloud:
|
|
result["ip_address"] = cloud.get("ip_address", "")
|
|
|
|
# CPU
|
|
cpu_data = state_meta.get("cpu", {})
|
|
cpu_usage_ns = cpu_data.get("usage", 0)
|
|
cpu_seconds = cpu_usage_ns / 1_000_000_000
|
|
result["cpu_usage"] = min(round(cpu_seconds % 100, 1), 100)
|
|
|
|
# Memory
|
|
mem_data = state_meta.get("memory", {})
|
|
mem_usage = mem_data.get("usage", 0)
|
|
mem_total = mem_data.get("total", 0)
|
|
result["memory_usage_bytes"] = mem_usage
|
|
result["memory_total_bytes"] = mem_total
|
|
if mem_total > 0:
|
|
result["ram_usage"] = round((mem_usage / mem_total) * 100, 1)
|
|
|
|
# Disk
|
|
disk_data = state_meta.get("disk", {})
|
|
root_disk = disk_data.get("root", {})
|
|
disk_usage = root_disk.get("usage", 0)
|
|
disk_total = root_disk.get("total", 0)
|
|
result["disk_usage_bytes"] = disk_usage
|
|
result["disk_total_bytes"] = disk_total
|
|
if disk_total > 0:
|
|
result["disk_usage"] = round((disk_usage / disk_total) * 100, 1)
|
|
|
|
# Network
|
|
network = state_meta.get("network", {})
|
|
for iface_name, iface in network.items():
|
|
if iface_name == "lo":
|
|
continue
|
|
counters = iface.get("counters", {})
|
|
result["network_rx"] += counters.get("bytes_received", 0)
|
|
result["network_tx"] += counters.get("bytes_sent", 0)
|
|
|
|
result["connections"] = state_meta.get("processes", 0)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
if cloud:
|
|
result["ip_address"] = cloud.get("ip_address", "")
|
|
|
|
return result
|
|
|
|
|
|
@frappe.whitelist()
|
|
def start_container(container_name, target_server=None):
|
|
"""Start a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "start",
|
|
"timeout": 30,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
|
|
return {"status": "ok", "message": f"Container {container_name} started"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def stop_container(container_name, target_server=None):
|
|
"""Stop a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "stop",
|
|
"timeout": 30,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
|
|
return {"status": "ok", "message": f"Container {container_name} stopped"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def restart_container(container_name, target_server=None):
|
|
"""Restart a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "restart",
|
|
"timeout": 30,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
|
|
return {"status": "ok", "message": f"Container {container_name} restarted"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def create_container(container_name, image="alpine", container_type="container", target_server=None):
|
|
"""Create a new container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
payload = {
|
|
"name": container_name,
|
|
"type": container_type,
|
|
"source": {
|
|
"type": "image",
|
|
"alias": image,
|
|
},
|
|
}
|
|
|
|
resp = _incus_api(target_server, "POST", "/1.0/instances", payload)
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=120)
|
|
|
|
# Auto-start the container
|
|
try:
|
|
start_resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "start",
|
|
"timeout": 30,
|
|
})
|
|
op = start_resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
except Exception:
|
|
pass
|
|
|
|
return {"status": "ok", "message": f"Container {container_name} created on {target_server}"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def delete_container(container_name, target_server=None):
|
|
"""Delete a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
# Stop first if running
|
|
try:
|
|
detail = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
if detail.get("metadata", {}).get("status") == "Running":
|
|
stop_resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "stop",
|
|
"timeout": 30,
|
|
"force": True,
|
|
})
|
|
op = stop_resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
except Exception:
|
|
pass
|
|
|
|
resp = _incus_api(target_server, "DELETE", f"/1.0/instances/{container_name}")
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
|
|
return {"status": "ok", "message": f"Container {container_name} deleted"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_container_logs(container_name, target_server=None):
|
|
"""Get container logs."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
return {"error": "Server name is required"}
|
|
|
|
config = _get_server_config(target_server)
|
|
|
|
try:
|
|
resp = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}/logs")
|
|
log_urls = resp.get("metadata", [])
|
|
|
|
logs = []
|
|
for log_url in log_urls:
|
|
log_name = log_url.rsplit("/", 1)[-1]
|
|
if not log_name.endswith(".log"):
|
|
continue
|
|
try:
|
|
r = requests.get(
|
|
f"{config['url']}/1.0/instances/{container_name}/logs/{log_name}",
|
|
cert=(config["cert"], config["key"]),
|
|
verify=False,
|
|
timeout=30,
|
|
)
|
|
r.raise_for_status()
|
|
content = r.text
|
|
for line in content.strip().split("\n")[-100:]:
|
|
if not line.strip():
|
|
continue
|
|
level = "info"
|
|
if "error" in line.lower() or "fail" in line.lower():
|
|
level = "error"
|
|
elif "warn" in line.lower():
|
|
level = "warning"
|
|
logs.append({
|
|
"source": log_name,
|
|
"level": level,
|
|
"message": line.strip(),
|
|
"timestamp": "",
|
|
})
|
|
except Exception:
|
|
continue
|
|
|
|
return logs
|
|
except Exception as e:
|
|
return [{"source": "system", "level": "error", "message": str(e), "timestamp": ""}]
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_snapshots(container_name, target_server=None):
|
|
"""Get list of snapshots for a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
return []
|
|
|
|
try:
|
|
resp = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}/snapshots?recursion=1")
|
|
snapshots = []
|
|
for snap in resp.get("metadata", []):
|
|
snapshots.append({
|
|
"name": snap.get("name", ""),
|
|
"created_at": snap.get("created_at", ""),
|
|
"stateful": snap.get("stateful", False),
|
|
"size": snap.get("size", 0),
|
|
})
|
|
snapshots.sort(key=lambda s: s["created_at"], reverse=True)
|
|
return snapshots
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
@frappe.whitelist()
|
|
def create_snapshot(container_name, snapshot_name, stateful=False, target_server=None):
|
|
"""Create a snapshot of a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
if isinstance(stateful, str):
|
|
stateful = stateful.lower() in ("true", "1")
|
|
|
|
resp = _incus_api(target_server, "POST", f"/1.0/instances/{container_name}/snapshots", {
|
|
"name": snapshot_name,
|
|
"stateful": stateful,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=120)
|
|
|
|
return {"status": "ok", "message": f"Snapshot {snapshot_name} created"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def restore_snapshot(container_name, snapshot_name, target_server=None):
|
|
"""Restore a container from a snapshot."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}", {
|
|
"restore": snapshot_name,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=120)
|
|
|
|
return {"status": "ok", "message": f"Container restored from snapshot {snapshot_name}"}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def delete_snapshot(container_name, snapshot_name, target_server=None):
|
|
"""Delete a snapshot."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
resp = _incus_api(target_server, "DELETE", f"/1.0/instances/{container_name}/snapshots/{snapshot_name}")
|
|
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
|
|
return {"status": "ok", "message": f"Snapshot {snapshot_name} deleted"}
|
|
|
|
|
|
def _exec_in_container(target_server, container_name, command, timeout=300):
|
|
"""Execute a command inside a container and return output."""
|
|
if isinstance(command, str):
|
|
command = ["sh", "-c", command]
|
|
|
|
resp = _incus_api(target_server, "POST", f"/1.0/instances/{container_name}/exec", {
|
|
"command": command,
|
|
"wait-for-websocket": False,
|
|
"interactive": False,
|
|
"record-output": True,
|
|
})
|
|
|
|
op = resp.get("operation")
|
|
if not op:
|
|
return {"return_code": -1, "stdout": "", "stderr": "No operation returned"}
|
|
|
|
result = _wait_for_operation(target_server, op, timeout=timeout)
|
|
|
|
output_lines = []
|
|
return_code = -1
|
|
|
|
if result:
|
|
meta = result.get("metadata", {})
|
|
return_code = meta.get("metadata", {}).get("return", -1)
|
|
|
|
# Try to fetch logged output
|
|
output = meta.get("output", {})
|
|
for key in ("1", "2"):
|
|
log_url = output.get(key, "")
|
|
if log_url:
|
|
try:
|
|
config = _get_server_config(target_server)
|
|
r = requests.get(
|
|
f"{config['url']}{log_url}",
|
|
cert=(config["cert"], config["key"]),
|
|
verify=False,
|
|
timeout=30,
|
|
)
|
|
if r.status_code == 200 and r.text.strip():
|
|
output_lines.append(r.text.strip())
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"return_code": return_code,
|
|
"output": "\n".join(output_lines),
|
|
}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def run_bench_migrate(container_name, target_server=None):
|
|
"""Run bench migrate inside a container."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
logs = []
|
|
logs.append("=== bench migrate ===")
|
|
logs.append(f"Container: {container_name}")
|
|
logs.append("")
|
|
|
|
try:
|
|
# Check container is running
|
|
detail = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
status = detail.get("metadata", {}).get("status")
|
|
if status != "Running":
|
|
logs.append(f"✗ Container status is '{status}', must be Running")
|
|
return {"success": False, "error": "Container must be running", "logs": logs}
|
|
|
|
# Check services are ready
|
|
logs.append("Checking database connection...")
|
|
if _wait_services_ready(target_server, container_name, timeout=30):
|
|
logs.append("✓ Database is ready")
|
|
else:
|
|
logs.append("⚠ Could not verify database, attempting migrate anyway")
|
|
|
|
logs.append("")
|
|
logs.append("Running: uvx --from frappe-bench bench migrate")
|
|
logs.append("")
|
|
|
|
result = _exec_in_container(
|
|
target_server, container_name,
|
|
"su frappe -c 'cd /home/frappe/frappe-bench && uvx --from frappe-bench bench migrate' 2>&1",
|
|
timeout=600,
|
|
)
|
|
|
|
output = result.get("output", "")
|
|
if output:
|
|
for line in output.split("\n"):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if "error" in line.lower() or "traceback" in line.lower() or "exception" in line.lower():
|
|
logs.append(f" ✗ {line}")
|
|
elif "migrating" in line.lower() or "patching" in line.lower():
|
|
logs.append(f" → {line}")
|
|
elif "success" in line.lower() or "done" in line.lower() or "completed" in line.lower():
|
|
logs.append(f" ✓ {line}")
|
|
else:
|
|
logs.append(f" {line}")
|
|
|
|
logs.append("")
|
|
if result["return_code"] == 0:
|
|
logs.append("✓ bench migrate completed successfully")
|
|
return {"success": True, "logs": logs, "message": "bench migrate completed successfully"}
|
|
else:
|
|
logs.append(f"✗ bench migrate failed (exit code {result['return_code']})")
|
|
if not output:
|
|
logs.append("")
|
|
logs.append("No output captured. Try running manually:")
|
|
logs.append(f" incus exec {container_name} -- su frappe -c 'cd /home/frappe/frappe-bench && uvx --from frappe-bench bench migrate'")
|
|
return {"success": False, "logs": logs, "error": "bench migrate failed"}
|
|
|
|
except Exception as e:
|
|
logs.append(f"✗ Error: {str(e)}")
|
|
return {"success": False, "error": str(e), "logs": logs}
|
|
|
|
|
|
def _wait_container_status(target_server, container_name, expected_status, timeout=60, poll_interval=2):
|
|
"""Poll container status until it matches expected_status or timeout."""
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
try:
|
|
detail = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
status = detail.get("metadata", {}).get("status", "")
|
|
if status == expected_status:
|
|
return True
|
|
except Exception:
|
|
pass
|
|
time.sleep(poll_interval)
|
|
return False
|
|
|
|
|
|
def _wait_services_ready(target_server, container_name, timeout=90, poll_interval=5):
|
|
"""Wait until MariaDB/MySQL is accepting connections inside container."""
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
try:
|
|
result = _exec_in_container(
|
|
target_server, container_name,
|
|
"su frappe -c 'cd /home/frappe/frappe-bench && python -c \"import frappe; frappe.init(site=frappe.utils.get_sites()[0]); frappe.connect(); print(frappe.db.sql(\\\"SELECT 1\\\")[0][0])\"' 2>&1",
|
|
timeout=30,
|
|
)
|
|
if result["return_code"] == 0 and "1" in result.get("output", ""):
|
|
return True
|
|
except Exception:
|
|
pass
|
|
time.sleep(poll_interval)
|
|
return False
|
|
|
|
|
|
@frappe.whitelist()
|
|
def rebuild_container(container_name, image, target_server=None, run_migrate=False):
|
|
"""Rebuild a container with a new image."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
if isinstance(run_migrate, str):
|
|
run_migrate = run_migrate.lower() in ("true", "1")
|
|
|
|
logs = []
|
|
migrate_success = True
|
|
logs.append(f"=== Update Container ===")
|
|
logs.append(f"Container: {container_name}")
|
|
logs.append(f"New image: {image}")
|
|
logs.append(f"Server: {target_server}")
|
|
logs.append(f"Run migrate: {'yes' if run_migrate else 'no'}")
|
|
logs.append("")
|
|
|
|
try:
|
|
# Step 1: Check current status
|
|
detail = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
meta = detail.get("metadata", {})
|
|
was_running = meta.get("status") == "Running"
|
|
logs.append(f"[1/5] Current status: {meta.get('status')}")
|
|
|
|
# Step 2: Stop container gracefully, then verify
|
|
if was_running:
|
|
logs.append("[2/5] Stopping container (graceful)...")
|
|
try:
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "stop",
|
|
"timeout": 60,
|
|
"force": False,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=90)
|
|
except Exception:
|
|
# Graceful stop failed, try force
|
|
logs.append(" Graceful stop timed out, forcing...")
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "stop",
|
|
"timeout": 30,
|
|
"force": True,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=60)
|
|
|
|
# Verify container actually stopped
|
|
if _wait_container_status(target_server, container_name, "Stopped", timeout=30):
|
|
logs.append(" ✓ Container stopped")
|
|
else:
|
|
logs.append(" ⚠ Container may not have fully stopped, proceeding anyway")
|
|
else:
|
|
logs.append("[2/5] Container already stopped, skipping")
|
|
|
|
# Step 3: Rebuild with new image
|
|
logs.append(f"[3/5] Rebuilding with image '{image}'...")
|
|
resp = _incus_api(target_server, "POST", f"/1.0/instances/{container_name}/rebuild", {
|
|
"source": {
|
|
"type": "image",
|
|
"alias": image,
|
|
},
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=300)
|
|
logs.append(" ✓ Container rebuilt successfully")
|
|
|
|
# Step 4: Start container and wait for services
|
|
if was_running:
|
|
logs.append("[4/5] Starting container...")
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "start",
|
|
"timeout": 30,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=60)
|
|
|
|
# Verify container is running
|
|
if _wait_container_status(target_server, container_name, "Running", timeout=30):
|
|
logs.append(" ✓ Container is running")
|
|
else:
|
|
logs.append(" ✗ Container failed to start")
|
|
logs.append("")
|
|
logs.append("=== Update finished with errors ===")
|
|
return {
|
|
"success": False,
|
|
"container_name": container_name,
|
|
"image": image,
|
|
"logs": logs,
|
|
"message": "Container failed to start after rebuild",
|
|
}
|
|
|
|
# Wait for services (DB, Redis, etc.)
|
|
if run_migrate:
|
|
logs.append(" Waiting for services (database, redis)...")
|
|
if _wait_services_ready(target_server, container_name, timeout=90):
|
|
logs.append(" ✓ Services are ready")
|
|
else:
|
|
logs.append(" ⚠ Services may not be fully ready, attempting migrate anyway")
|
|
else:
|
|
logs.append("[4/5] Container was stopped, skipping start")
|
|
|
|
# Step 5: Run bench migrate
|
|
if run_migrate and was_running:
|
|
logs.append("[5/5] Running bench migrate...")
|
|
logs.append("")
|
|
migrate_result = _exec_in_container(
|
|
target_server, container_name,
|
|
"su frappe -c 'cd /home/frappe/frappe-bench && uvx --from frappe-bench bench migrate' 2>&1",
|
|
timeout=600,
|
|
)
|
|
|
|
output = migrate_result.get("output", "")
|
|
if output:
|
|
# Parse and format output nicely
|
|
for line in output.split("\n"):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
# Highlight important lines
|
|
if "error" in line.lower() or "traceback" in line.lower() or "exception" in line.lower():
|
|
logs.append(f" ✗ {line}")
|
|
elif "migrating" in line.lower() or "patching" in line.lower():
|
|
logs.append(f" → {line}")
|
|
elif "success" in line.lower() or "done" in line.lower() or "completed" in line.lower():
|
|
logs.append(f" ✓ {line}")
|
|
else:
|
|
logs.append(f" {line}")
|
|
|
|
logs.append("")
|
|
if migrate_result["return_code"] == 0:
|
|
logs.append(" ✓ bench migrate completed successfully")
|
|
else:
|
|
migrate_success = False
|
|
logs.append(f" ✗ bench migrate failed (exit code {migrate_result['return_code']})")
|
|
if not output:
|
|
logs.append(" No output captured. The command may have crashed.")
|
|
logs.append(" Try running manually inside the container:")
|
|
logs.append(f" incus exec {container_name} -- su frappe -c 'cd /home/frappe/frappe-bench && uvx --from frappe-bench bench migrate'")
|
|
else:
|
|
logs.append("[5/5] Skipping migrate" + (" (container was stopped)" if not was_running else ""))
|
|
|
|
logs.append("")
|
|
if migrate_success:
|
|
logs.append("=== Update completed successfully! ===")
|
|
return {
|
|
"success": True,
|
|
"container_name": container_name,
|
|
"image": image,
|
|
"logs": logs,
|
|
"message": f"Container '{container_name}' updated with image '{image}'",
|
|
}
|
|
else:
|
|
logs.append("=== Update completed, but migrate failed ===")
|
|
return {
|
|
"success": False,
|
|
"container_name": container_name,
|
|
"image": image,
|
|
"logs": logs,
|
|
"message": f"Container rebuilt, but bench migrate failed. Check logs for details.",
|
|
}
|
|
|
|
except Exception as e:
|
|
logs.append(f"✗ Error: {str(e)}")
|
|
return {
|
|
"success": False,
|
|
"error": str(e),
|
|
"logs": logs,
|
|
"message": f"Update failed: {str(e)}",
|
|
}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def deploy_container_full(container_name, image="erp", pool_name="default", volumes_config=None, target_server=None):
|
|
"""Full container deploy with volumes."""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
if isinstance(volumes_config, str):
|
|
volumes_config = json.loads(volumes_config)
|
|
|
|
if not volumes_config:
|
|
volumes_config = DEFAULT_VOLUMES
|
|
|
|
logs = []
|
|
current_step = 0
|
|
tmp_mount_path = "/mnt/tmp"
|
|
|
|
volumes_info = []
|
|
for vol_config in volumes_config:
|
|
volumes_info.append({
|
|
"volume_name": f"{container_name}-{vol_config['name']}",
|
|
"source_path": vol_config["source_path"],
|
|
"device_name": vol_config["device_name"],
|
|
"temp_device_name": f"tmp_{vol_config['name']}",
|
|
})
|
|
|
|
logs.append(f"=== Deploy Configuration ===")
|
|
logs.append(f"Container: {container_name}")
|
|
logs.append(f"Image: {image}")
|
|
logs.append(f"Server: {target_server}")
|
|
logs.append(f"Volumes: {len(volumes_info)}")
|
|
|
|
try:
|
|
# Step 1: Create container
|
|
current_step = 1
|
|
logs.append("Step 1: Creating container...")
|
|
payload = {
|
|
"name": container_name,
|
|
"type": "container",
|
|
"source": {"type": "image", "alias": image},
|
|
}
|
|
resp = _incus_api(target_server, "POST", "/1.0/instances", payload)
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=120)
|
|
logs.append(f"✓ Container '{container_name}' created")
|
|
|
|
# Step 2: Start container
|
|
current_step = 2
|
|
logs.append("Step 2: Starting container...")
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "start", "timeout": 30,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
time.sleep(3)
|
|
logs.append(f"✓ Container '{container_name}' started")
|
|
|
|
# Step 3: Create volumes
|
|
current_step = 3
|
|
logs.append(f"Step 3: Creating {len(volumes_info)} volume(s)...")
|
|
for vol_info in volumes_info:
|
|
try:
|
|
_incus_api(target_server, "POST", f"/1.0/storage-pools/{pool_name}/volumes", {
|
|
"name": vol_info["volume_name"],
|
|
"type": "custom",
|
|
"config": {},
|
|
})
|
|
logs.append(f" ✓ Volume '{vol_info['volume_name']}' created")
|
|
except Exception as e:
|
|
if "409" in str(e):
|
|
logs.append(f" - Volume '{vol_info['volume_name']}' already exists")
|
|
else:
|
|
raise
|
|
|
|
# Step 4: Temporarily mount volumes
|
|
current_step = 4
|
|
logs.append(f"Step 4: Mounting volumes temporarily...")
|
|
for i, vol_info in enumerate(volumes_info):
|
|
temp_path = f"{tmp_mount_path}{i+1}"
|
|
vol_info["temp_path"] = temp_path
|
|
temp_device_config = {
|
|
"type": "disk",
|
|
"pool": pool_name,
|
|
"source": vol_info["volume_name"],
|
|
"path": temp_path,
|
|
}
|
|
resp = _incus_api(target_server, "PATCH", f"/1.0/instances/{container_name}", {
|
|
"devices": {vol_info["temp_device_name"]: temp_device_config},
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
logs.append(f" ✓ Mounted at '{temp_path}'")
|
|
time.sleep(2)
|
|
|
|
# Step 5: Copy data
|
|
current_step = 5
|
|
logs.append(f"Step 5: Copying data to volumes...")
|
|
for vol_info in volumes_info:
|
|
exec_payload = {
|
|
"command": ["cp", "-a", f"{vol_info['source_path']}/.", f"{vol_info['temp_path']}/"],
|
|
"wait-for-websocket": False,
|
|
"interactive": False,
|
|
}
|
|
try:
|
|
resp = _incus_api(target_server, "POST", f"/1.0/instances/{container_name}/exec", exec_payload)
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op, timeout=120)
|
|
logs.append(f" ✓ Copied {vol_info['source_path']}")
|
|
except Exception as e:
|
|
logs.append(f" - Warning copying {vol_info['source_path']}: {str(e)}")
|
|
|
|
# Step 6: Unmount temporary volumes
|
|
current_step = 6
|
|
logs.append(f"Step 6: Unmounting temporary volumes...")
|
|
current = _incus_api(target_server, "GET", f"/1.0/instances/{container_name}")
|
|
current_meta = current.get("metadata", {})
|
|
devices = current_meta.get("devices", {})
|
|
for vol_info in volumes_info:
|
|
devices.pop(vol_info["temp_device_name"], None)
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}", {
|
|
"devices": devices,
|
|
"config": current_meta.get("config", {}),
|
|
"profiles": current_meta.get("profiles", []),
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
logs.append(f"✓ Temporary mounts removed")
|
|
|
|
# Step 7: Stop container
|
|
current_step = 7
|
|
logs.append("Step 7: Stopping container...")
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "stop", "timeout": 30,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
logs.append(f"✓ Container stopped")
|
|
|
|
# Step 8: Mount volumes permanently
|
|
current_step = 8
|
|
logs.append(f"Step 8: Mounting volumes permanently...")
|
|
for vol_info in volumes_info:
|
|
permanent_device_config = {
|
|
"type": "disk",
|
|
"pool": pool_name,
|
|
"source": vol_info["volume_name"],
|
|
"path": vol_info["source_path"],
|
|
}
|
|
resp = _incus_api(target_server, "PATCH", f"/1.0/instances/{container_name}", {
|
|
"devices": {vol_info["device_name"]: permanent_device_config},
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
logs.append(f" ✓ {vol_info['volume_name']} → {vol_info['source_path']}")
|
|
|
|
# Step 9: Start container
|
|
current_step = 9
|
|
logs.append("Step 9: Starting container (final)...")
|
|
resp = _incus_api(target_server, "PUT", f"/1.0/instances/{container_name}/state", {
|
|
"action": "start", "timeout": 30,
|
|
})
|
|
op = resp.get("operation")
|
|
if op:
|
|
_wait_for_operation(target_server, op)
|
|
logs.append(f"✓ Container '{container_name}' started")
|
|
|
|
logs.append("\n=== Deploy completed successfully! ===")
|
|
return {
|
|
"success": True,
|
|
"container_name": container_name,
|
|
"volumes": [v["volume_name"] for v in volumes_info],
|
|
"logs": logs,
|
|
"message": f"Container '{container_name}' deployed successfully",
|
|
}
|
|
|
|
except Exception as e:
|
|
logs.append(f"✗ Error at step {current_step}: {str(e)}")
|
|
return {
|
|
"success": False,
|
|
"step": current_step,
|
|
"error": str(e),
|
|
"logs": logs,
|
|
"message": f"Deploy failed at step {current_step}: {str(e)}",
|
|
}
|
|
|
|
|
|
@frappe.whitelist()
|
|
def get_login_url(container_name, target_server=None):
|
|
"""Generate a login URL for a container site (login as Administrator).
|
|
|
|
Uses direct HTTP POST to the site's /api/method/login endpoint,
|
|
same approach as Frappe Cloud (press).
|
|
"""
|
|
_check_admin_permission()
|
|
|
|
if not target_server:
|
|
frappe.throw("Server name is required")
|
|
|
|
host_domain = frappe.conf.get("host_domain", "host.jeyerp.az")
|
|
site_url = f"http://{container_name}.{host_domain}"
|
|
|
|
# POST to site's login API with admin credentials (like Frappe Press)
|
|
try:
|
|
response = requests.post(
|
|
f"{site_url}/api/method/login",
|
|
data={"usr": "Administrator", "pwd": "admin"},
|
|
timeout=10,
|
|
)
|
|
except requests.exceptions.ConnectionError:
|
|
return {"success": False, "error": f"Cannot connect to {site_url}"}
|
|
except requests.exceptions.Timeout:
|
|
return {"success": False, "error": f"Connection to {site_url} timed out"}
|
|
|
|
sid = response.cookies.get("sid")
|
|
|
|
if not sid or sid == "Guest":
|
|
return {
|
|
"success": False,
|
|
"error": "Login failed. Check that site is running and admin password is correct.",
|
|
}
|
|
|
|
return {
|
|
"success": True,
|
|
"url": site_url,
|
|
"sid": sid,
|
|
}
|