Graceful Multi-Tenant Lifecycle: Managing Isolated Containers, Drains, and Deferred Purges in Python

# ❌ DANGEROUS: Immediately sends SIGKILL (Signal 9)
container.remove(v=True, force=True)

def stop_container(self, container_name: str, remove: bool = True, timeout: int = 30):
    if not self.client:
        return
    try:
        container = self.client.containers.get(container_name)

        # 1. Send SIGTERM and grant a 30s grace window
        container.stop(timeout=timeout)
        logger.info(f"Container {container_name} stopped gracefully.")

        # 2. Clean up container and associated volumes safely
        if remove:
            container.remove(v=True)
            logger.info(f"Container {container_name} and volumes purged.")

    except docker.errors.NotFound:
        logger.warning(f"Container {container_name} not found, skipping.")
    except Exception as e:
        logger.error(f"Error stopping/removing {container_name}: {e}")

async def provision(self, tenant: Tenant, db: AsyncSession):
    # Tenant-level lock prevents duplicate concurrent provisioning
    async with OrchestrationLock(settings.REDIS_URL) as locker:
        lock_name = locker.tenant_provision_lock(tenant.id)
        if not await locker.lock.acquire(lock_name):
            raise RuntimeError(f"Provisioning already in progress for tenant {tenant.id}")

        try:
            return await self._execute_provisioning_flow(tenant, db)
        except Exception as e:
            logger.error(f"Provisioning failed for {tenant.subdomain}: {e}")
            await self._rollback()
            raise
        finally:
            await locker.lock.release(lock_name)

[1. Allocate DB] ➔ [2. Render Configs] ➔ [3. Spin Docker] ➔ [4. Healthcheck] ➔ [5. Seed State]

class ProvisioningOrchestrator:
    def __init__(self):
        self._rollback_stack: List[Callable[[], Coroutine[Any, Any, None]]] = []

    async def _execute_provisioning_flow(self, tenant: Tenant, db: AsyncSession):
        self._rollback_stack = []

        # Step 1: Create isolated PostgreSQL tenant DB
        db_success = await self.postgres.create_tenant_db(db_name, db_user, db_pass)
        if not db_success:
            raise RuntimeError("Database creation failed")

        # Register rollback handler for Step 1
        self._rollback_stack.append(
            lambda: self.postgres.drop_tenant_db(db_name, db_user)
        )

        # Step 2: Spin up container using CLI subprocess
        await self.docker.start_instance(tenant_dir)

        # Register rollback handler for Step 2
        self._rollback_stack.append(
            lambda: self._async_wrapper(self.docker.stop_container, container_name)
        )

        # Step 3: Wait for Healthcheck
        if not await self.wait_for_health(internal_url):
            raise RuntimeError("Instance failed healthcheck")

    async def _rollback(self):
        logger.warning(f"Initiating rollback for {len(self._rollback_stack)} steps...")
        # Execute registered rollbacks in reverse order
        for rollback_task in reversed(self._rollback_stack):
            try:
                await rollback_task()
            except Exception as e:
                logger.error(f"Rollback step failed: {e}")

async def _async_wrapper(self, sync_fn, *args, **kwargs):
    # Offloads blocking Docker SDK calls to a background worker thread
    return await asyncio.to_thread(sync_fn, *args, **kwargs)

SUBSCRIPTION_FREEZE_DAYS = 7   # Soft Stop: Release RAM/CPU compute
SUBSCRIPTION_PURGE_DAYS = 365  # Hard Purge: Delete Database & Disk Files

[Active Tenant] ──(Subscription Lapses)──> [Suspended] ──(Day 7 Expiry)──> [Frozen (0 MB RAM)] ──(Day 365 Expiry)──> [Hard Purged]
                                            Container stopped/removed.       PostgreSQL DB dropped.
                                            PostgreSQL DB preserved.         Filesystem wiped.

async def handle_frozen_tenants(db: AsyncSession):
    """Cron job: Offloads compute containers for tenants inactive > 7 days."""
    freeze_threshold = datetime.utcnow() - timedelta(days=settings.SUBSCRIPTION_FREEZE_DAYS)
    inactive_tenants = await tenant_crud.get_tenants_eligible_for_freeze(db, freeze_threshold)

    for tenant in inactive_tenants:
        instance = await listmonk_crud.get_instance_by_tenant(db, tenant.id)
        if instance and instance.container_name:
            # Stop and remove container, but preserve DB and files
            await asyncio.to_thread(orchestrator.docker.stop_container, instance.container_name, remove=True)
            await listmonk_crud.update_instance_status(db, instance.id, "frozen")
            logger.info(f"Tenant {tenant.subdomain} container frozen. Compute freed.")

1. Remove Docker Container (Stop incoming traffic & queues)
       │
       ▼
2. Drop PostgreSQL DB (Clean storage once process handles detach)
       │
       ▼
3. Purge File Directory (Wipe local compose configs and assets)
       │
       ▼
4. Update Control Plane DB (Commit deactivated status)

async def deprovision(self, tenant: Tenant, db: AsyncSession) -> Dict[str, Any]:
    instance = await listmonk_crud.get_instance_by_tenant(db, tenant.id)
    if not instance:
        tenant.status = "deactivated"
        await db.commit()
        return {"status": "not_provisioned"}

    cleanup_errors = []

    # 1. Stop and remove container first
    if instance.container_name:
        try:
            await asyncio.to_thread(self.docker.stop_container, instance.container_name, remove=True)
        except Exception as e:
            cleanup_errors.append(f"docker_container: {e}")

    # 2. Drop the isolated PostgreSQL database
    if instance.database_name and instance.database_user:
        try:
            await self.postgres.drop_tenant_db(instance.database_name, instance.database_user)
        except Exception as e:
            cleanup_errors.append(f"postgres_db: {e}")

    # 3. Clean files off the disk
    tenant_dir = os.path.abspath(self.config_svc.get_tenant_dir(tenant.subdomain))
    try:
        if os.path.exists(tenant_dir):
            shutil.rmtree(tenant_dir)
    except Exception as e:
        cleanup_errors.append(f"disk_directory: {e}")

    # 4. Clean up relational metadata & commit
    await db.execute(delete(SMTPConfig).where(SMTPConfig.tenant_id == tenant.id))
    await listmonk_crud.update_instance_status(db, instance.id, "deleted")

    tenant.status = "deactivated"
    tenant.listmonk_api_url = None
    await db.commit()

    if cleanup_errors:
        logger.warning(f"Tenant {tenant.subdomain} deprovisioned with partial failures: {cleanup_errors}")
    else:
        logger.info(f"Tenant {tenant.subdomain} fully purged after {settings.SUBSCRIPTION_PURGE_DAYS} days.")

    return {"status": "deactivated"}