Coverage for mcpgateway / tools / builder / pipeline.py: 100%

1# -*- coding: utf-8 -*- 

2"""Location: ./mcpgateway/tools/builder/pipeline.py 

3Copyright 2025 

4SPDX-License-Identifier: Apache-2.0 

5Authors: Teryl Taylor 

6 

7Abstract base class for MCP Stack deployment implementations. 

8 

9This module defines the CICDModule interface that all deployment implementations 

10must implement. It provides a common API for building, deploying, and managing 

11MCP Gateway stacks with external plugin servers. 

12 

13The base class implements shared functionality (validation) while requiring 

14subclasses to implement deployment-specific logic (build, deploy, etc.). 

15 

16Design Pattern: 

17 Strategy Pattern - Different implementations (Dagger vs Python) can be 

18 swapped transparently via the DeployFactory. 

19 

20Example: 

21 >>> from mcpgateway.tools.builder.factory import DeployFactory 

22 >>> deployer, mode = DeployFactory.create_deployer("dagger", verbose=False) 

23 ⚠ Dagger not installed. Using plain python. 

24 >>> # Validate configuration (output varies by config) 

25 >>> # deployer.validate("mcp-stack.yaml") 

26 >>> # Async methods must be called with await (see method examples below) 

27""" 

28 

29# Standard 

30from abc import ABC, abstractmethod 

31from pathlib import Path 

32from typing import Optional 

33 

34# Third-Party 

35from pydantic import ValidationError 

36from rich.console import Console 

37import yaml 

38 

39# First-Party 

40from mcpgateway.tools.builder.schema import MCPStackConfig 

41 

42# Shared console instance for consistent output formatting 

43console = Console() 

44 

45 

46class CICDModule(ABC): 

47 """Abstract base class for MCP Stack deployment implementations. 

48 

49 This class defines the interface that all deployment implementations must 

50 implement. It provides common initialization and validation logic while 

51 deferring implementation-specific details to subclasses. 

52 

53 Attributes: 

54 verbose (bool): Enable verbose output during operations 

55 console (Console): Rich console for formatted output 

56 

57 Implementations: 

58 - MCPStackDagger: High-performance implementation using Dagger SDK 

59 - MCPStackPython: Fallback implementation using plain Python + Docker/Podman 

60 

61 Examples: 

62 >>> # Test that CICDModule is abstract 

63 >>> from abc import ABC 

64 >>> issubclass(CICDModule, ABC) 

65 True 

66 

67 >>> # Test initialization with defaults 

68 >>> class TestDeployer(CICDModule): 

69 ... async def build(self, config_file: str, **kwargs) -> None: 

70 ... pass 

71 ... async def generate_certificates(self, config_file: str) -> None: 

72 ... pass 

73 ... async def deploy(self, config_file: str, **kwargs) -> None: 

74 ... pass 

75 ... async def verify(self, config_file: str, **kwargs) -> None: 

76 ... pass 

77 ... async def destroy(self, config_file: str) -> None: 

78 ... pass 

79 ... def generate_manifests(self, config_file: str, **kwargs) -> Path: 

80 ... return Path(".") 

81 >>> deployer = TestDeployer() 

82 >>> deployer.verbose 

83 False 

84 

85 >>> # Test initialization with verbose=True 

86 >>> verbose_deployer = TestDeployer(verbose=True) 

87 >>> verbose_deployer.verbose 

88 True 

89 

90 >>> # Test that console is available 

91 >>> hasattr(deployer, 'console') 

92 True 

93 """ 

94 

95 def __init__(self, verbose: bool = False): 

96 """Initialize the deployment module. 

97 

98 Args: 

99 verbose: Enable verbose output during all operations 

100 

101 Examples: 

102 >>> # Cannot instantiate abstract class directly 

103 >>> try: 

104 ... CICDModule() 

105 ... except TypeError as e: 

106 ... "abstract" in str(e).lower() 

107 True 

108 """ 

109 self.verbose = verbose 

110 self.console = console 

111 

112 def validate(self, config_file: str) -> None: 

113 """Validate mcp-stack.yaml configuration using Pydantic schemas. 

114 

115 This method provides comprehensive validation of the MCP stack configuration 

116 using Pydantic models defined in schema.py. It validates: 

117 - Required sections (deployment, gateway, plugins) 

118 - Deployment type (kubernetes or compose) 

119 - Gateway image specification 

120 - Plugin configurations (name, repo/image, etc.) 

121 - Custom business rules (unique names, valid combinations) 

122 

123 Args: 

124 config_file: Path to mcp-stack.yaml configuration file 

125 

126 Raises: 

127 ValueError: If configuration is invalid, with formatted error details 

128 ValidationError: If Pydantic schema validation fails 

129 FileNotFoundError: If config_file does not exist 

130 

131 Examples: 

132 >>> import tempfile 

133 >>> import yaml 

134 >>> from pathlib import Path 

135 >>> # Create a test deployer 

136 >>> class TestDeployer(CICDModule): 

137 ... async def build(self, config_file: str, **kwargs) -> None: 

138 ... pass 

139 ... async def generate_certificates(self, config_file: str) -> None: 

140 ... pass 

141 ... async def deploy(self, config_file: str, **kwargs) -> None: 

142 ... pass 

143 ... async def verify(self, config_file: str, **kwargs) -> None: 

144 ... pass 

145 ... async def destroy(self, config_file: str) -> None: 

146 ... pass 

147 ... def generate_manifests(self, config_file: str, **kwargs) -> Path: 

148 ... return Path(".") 

149 >>> deployer = TestDeployer(verbose=False) 

150 

151 >>> # Test with valid minimal config 

152 >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: 

153 ... config = { 

154 ... 'deployment': {'type': 'compose'}, 

155 ... 'gateway': {'image': 'test:latest'}, 

156 ... 'plugins': [] 

157 ... } 

158 ... yaml.dump(config, f) 

159 ... config_path = f.name 

160 >>> deployer.validate(config_path) 

161 >>> import os 

162 >>> os.unlink(config_path) 

163 

164 >>> # Test with missing file 

165 >>> try: 

166 ... deployer.validate("/nonexistent/config.yaml") 

167 ... except FileNotFoundError as e: 

168 ... "config.yaml" in str(e) 

169 True 

170 

171 >>> # Test with invalid config (missing required fields) 

172 >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: 

173 ... bad_config = {'deployment': {'type': 'compose'}} 

174 ... yaml.dump(bad_config, f) 

175 ... bad_path = f.name 

176 >>> try: 

177 ... deployer.validate(bad_path) 

178 ... except ValueError as e: 

179 ... "validation failed" in str(e).lower() 

180 True 

181 >>> os.unlink(bad_path) 

182 """ 

183 if self.verbose: 

184 self.console.print(f"[blue]Validating {config_file}...[/blue]") 

185 

186 # Load YAML configuration 

187 with open(config_file, "r") as f: 

188 config_dict = yaml.safe_load(f) 

189 

190 # Validate using Pydantic schema 

191 try: 

192 # Local 

193 

194 MCPStackConfig(**config_dict) 

195 except ValidationError as e: 

196 # Format validation errors for better readability 

197 error_msg = "Configuration validation failed:\n" 

198 for error in e.errors(): 

199 # Join the error location path (e.g., plugins -> 0 -> name) 

200 loc = " -> ".join(str(x) for x in error["loc"]) 

201 error_msg += f" • {loc}: {error['msg']}\n" 

202 raise ValueError(error_msg) from e 

203 

204 if self.verbose: 

205 self.console.print("[green]✓ Configuration valid[/green]") 

206 

207 @abstractmethod 

208 async def build(self, config_file: str, plugins_only: bool = False, specific_plugins: Optional[list[str]] = None, no_cache: bool = False, copy_env_templates: bool = False) -> None: 

209 """Build container images for plugins and/or gateway. 

210 

211 Subclasses must implement this to build Docker/Podman images from 

212 Git repositories or use pre-built images. 

213 

214 Args: 

215 config_file: Path to mcp-stack.yaml 

216 plugins_only: Only build plugins, skip gateway 

217 specific_plugins: List of specific plugin names to build (optional) 

218 no_cache: Disable build cache for fresh builds 

219 copy_env_templates: Copy .env.template files from cloned repos 

220 

221 Raises: 

222 RuntimeError: If build fails 

223 ValueError: If plugin configuration is invalid 

224 

225 Example: 

226 # await deployer.build("mcp-stack.yaml", plugins_only=True) 

227 # ✓ Built OPAPluginFilter 

228 # ✓ Built LLMGuardPlugin 

229 """ 

230 

231 @abstractmethod 

232 async def generate_certificates(self, config_file: str) -> None: 

233 """Generate mTLS certificates for gateway and plugins. 

234 

235 Creates a certificate authority (CA) and issues certificates for: 

236 - Gateway (client certificates for connecting to plugins) 

237 - Each plugin (server certificates for accepting connections) 

238 

239 Certificates are stored in the paths defined in the config's 

240 certificates section (default: ./certs/mcp/). 

241 

242 Args: 

243 config_file: Path to mcp-stack.yaml 

244 

245 Raises: 

246 RuntimeError: If certificate generation fails 

247 FileNotFoundError: If required tools (openssl) are not available 

248 

249 Example: 

250 # await deployer.generate_certificates("mcp-stack.yaml") 

251 # ✓ Certificates generated 

252 """ 

253 

254 @abstractmethod 

255 async def deploy(self, config_file: str, dry_run: bool = False, skip_build: bool = False, skip_certs: bool = False) -> None: 

256 """Deploy the MCP stack to Kubernetes or Docker Compose. 

257 

258 This is the main deployment method that orchestrates: 

259 1. Building containers (unless skip_build=True) 

260 2. Generating mTLS certificates (unless skip_certs=True or mTLS disabled) 

261 3. Generating manifests (Kubernetes YAML or docker-compose.yaml) 

262 4. Applying the deployment (unless dry_run=True) 

263 

264 Args: 

265 config_file: Path to mcp-stack.yaml 

266 dry_run: Generate manifests without actually deploying 

267 skip_build: Skip building containers (use existing images) 

268 skip_certs: Skip certificate generation (use existing certs) 

269 

270 Raises: 

271 RuntimeError: If deployment fails at any stage 

272 ValueError: If configuration is invalid 

273 

274 Example: 

275 # Full deployment 

276 # await deployer.deploy("mcp-stack.yaml") 

277 # ✓ Build complete 

278 # ✓ Certificates generated 

279 # ✓ Deployment complete 

280 

281 # Dry run (generate manifests only) 

282 # await deployer.deploy("mcp-stack.yaml", dry_run=True) 

283 # ✓ Dry-run complete (no changes made) 

284 """ 

285 

286 @abstractmethod 

287 async def verify(self, config_file: str, wait: bool = False, timeout: int = 300) -> None: 

288 """Verify deployment health and readiness. 

289 

290 Checks that all deployed services are healthy and ready: 

291 - Kubernetes: Checks pod status, optionally waits for Ready 

292 - Docker Compose: Checks container status 

293 

294 Args: 

295 config_file: Path to mcp-stack.yaml 

296 wait: Wait for deployment to become ready 

297 timeout: Maximum time to wait in seconds (default: 300) 

298 

299 Raises: 

300 RuntimeError: If verification fails or timeout is reached 

301 TimeoutError: If wait=True and deployment doesn't become ready 

302 

303 Example: 

304 # Quick health check 

305 # await deployer.verify("mcp-stack.yaml") 

306 # NAME READY STATUS RESTARTS AGE 

307 # mcpgateway-xxx 1/1 Running 0 2m 

308 # mcp-plugin-opa-xxx 1/1 Running 0 2m 

309 

310 # Wait for ready state 

311 # await deployer.verify("mcp-stack.yaml", wait=True, timeout=600) 

312 # ✓ Deployment healthy 

313 """ 

314 

315 @abstractmethod 

316 async def destroy(self, config_file: str) -> None: 

317 """Destroy the deployed MCP stack. 

318 

319 Removes all deployed resources: 

320 - Kubernetes: Deletes all resources in the namespace 

321 - Docker Compose: Stops and removes containers, networks, volumes 

322 

323 WARNING: This is destructive and cannot be undone! 

324 

325 Args: 

326 config_file: Path to mcp-stack.yaml 

327 

328 Raises: 

329 RuntimeError: If destruction fails 

330 

331 Example: 

332 # await deployer.destroy("mcp-stack.yaml") 

333 # ✓ Deployment destroyed 

334 """ 

335 

336 @abstractmethod 

337 def generate_manifests(self, config_file: str, output_dir: Optional[str] = None) -> Path: 

338 """Generate deployment manifests (Kubernetes YAML or docker-compose.yaml). 

339 

340 Creates deployment manifests based on configuration: 

341 - Kubernetes: Generates Deployment, Service, ConfigMap, Secret YAML files 

342 - Docker Compose: Generates docker-compose.yaml with all services 

343 

344 Also generates: 

345 - plugins-config.yaml: Plugin manager configuration for gateway 

346 - Environment files: .env files for each service 

347 

348 Args: 

349 config_file: Path to mcp-stack.yaml 

350 output_dir: Output directory for manifests (default: ./deploy/manifests) 

351 

352 Returns: 

353 Path: Directory containing generated manifests 

354 

355 Raises: 

356 ValueError: If configuration is invalid 

357 OSError: If output directory cannot be created 

358 

359 Example: 

360 # manifests_path = deployer.generate_manifests("mcp-stack.yaml") 

361 # print(f"Manifests generated in: {manifests_path}") 

362 # Manifests generated in: /path/to/deploy/manifests 

363 

364 # Custom output directory 

365 # deployer.generate_manifests("mcp-stack.yaml", output_dir="./my-manifests") 

366 # ✓ Manifests generated: ./my-manifests 

367 """