model { kdsai = system 'KDS.AI' { description 'AI-powered kitchen display system for restaurants' frontend = frontend 'Frontend' { description 'Dashboard for the managers' } aibackend = service 'AI Backend' { description ' Backend for AI orchestration. This will be split into multiple services at some point ' technology '.NET 10 + Microsoft Agents Framework' style { icon tech:dotnet } } warehouseDb = redshift 'Tenant Redshift Databases' { description ' Logical analytics warehouse with one Redshift database per tenant. Each tenant database contains staging, curated, and serving schemas. AI queries are routed to the tenant-selected database and serving views only. ' technology 'AWS Redshift Serverless' } operationalDb = postgres 'Operational Database' { description 'Database for the application operations' } bedrock = service 'Bedrock' { description 'LLM hosting and inference platform' technology 'AWS Bedrock' style { color amber icon @/bedrock-color.svg } } // --- ETL Pipeline Components --- etl = system 'ETL Pipeline' { description ' Default customer data ingestion pipeline. Lands raw payloads in S3, prepares load files, routes batches into the correct tenant Redshift database, loads staging, and builds curated tables and serving views for monitoring. ' ingestionApi = service 'Ingestion API' { description ' Thin .NET API for file uploads and API push ingestion. Authenticates tenants, records batch metadata, stores raw payloads, and enqueues ETL work. ' technology '.NET 10 ASP.NET Core' } etlWorker = worker 'ETL Worker' { description ' .NET Aspire BackgroundService. Polls scheduled APIs, runs CDC snapshot or change extraction, validates payloads, prepares load files, resolves the target tenant database, loads tenant staging, and runs curated transforms plus serving view refreshes. ' technology '.NET 10 BackgroundService' } rawStorage = storage 'S3 Raw Storage' { description ' Durable landing zone for original customer payloads, raw API responses, and CDC snapshots or change batches. Also stores failed batches for replay and investigation. ' technology 'AWS S3' } loadStorage = storage 'S3 Load Files' { description ' Normalized files written by the worker and loaded into tenant Redshift staging schemas. Separates raw payload retention from warehouse load artifacts. ' technology 'AWS S3' } curatedStorage = storage 'S3 Curated Storage' { description ' Parquet files UNLOADed from Redshift curated tables. Partitioned by tenant_id, entity, and date. Serves as the query surface for cross-tenant Athena queries and long-term curated data retention. ' technology 'AWS S3' } ingestionQueue = queue 'Ingestion Queue' { description ' Decouples uploads, pushed payloads, and other ETL jobs from execution. ' technology 'AWS SQS' } } // --- Cross-Tenant Query Layer --- glueCatalog = catalog 'Glue Data Catalog' { description ' Metadata catalog with partition-projection tables over curated S3 Parquet. kdsai_crossstore database with tables partitioned by tenant_id. Eliminates need for crawlers via partition projection configuration. ' technology 'AWS Glue Data Catalog' } athena = service 'Athena' { description ' Serverless cross-tenant query engine. Used when an AI agent needs to query across multiple tenants (e.g. chain or district manager scenarios). Reads curated Parquet from S3 via Glue Data Catalog definitions. Platform injects WHERE tenant_id IN (...) for access control. ' technology 'AWS Athena' style { color amber icon tech:aws } } } // --- Core Relationships --- user -> frontend 'Uses' frontend -> aibackend 'Gets data from' frontend -> operationalDb 'Reads from and writes to' aibackend -> warehouseDb 'Queries tenant-selected serving views in' aibackend -> athena 'Queries cross-tenant curated data via' aibackend -> operationalDb 'Uses for operational data and AI knowledge base' aibackend -> bedrock 'Uses LLM models from' admin -> warehouseDb 'Supplies tenant warehouse data to (manual process for PoC)' admin -> operationalDb 'Supplies monitor specifications and knowledge base data to (manual process for PoC)' admin -> frontend 'Uses chatbot for use-case creation' aibackend -[notification]-> user 'Sends notifications to (e.g. monitoring alerts, etc.)' // --- ETL Internal Relationships --- etl.ingestionApi -> etl.rawStorage 'Stores raw payloads in' etl.ingestionApi -> etl.ingestionQueue 'Enqueues load jobs in' etl.etlWorker -> etl.ingestionQueue 'Consumes load jobs from' etl.etlWorker -> etl.rawStorage 'Reads raw payloads and writes failures to' etl.etlWorker -> etl.loadStorage 'Writes load-ready files to' etl.etlWorker -[etl]-> warehouseDb 'Loads tenant staging and builds curated/serving objects in' warehouseDb -[etl]-> etl.loadStorage 'Tenant databases COPY load staging data from' // --- UNLOAD & Cross-Tenant Relationships --- warehouseDb -[etl]-> etl.curatedStorage 'UNLOADs curated tables as Parquet to' glueCatalog -> etl.curatedStorage 'Partition-projection table definitions point to' athena -> glueCatalog 'Resolves table schemas and partitions from' athena -> etl.curatedStorage 'Reads curated Parquet files from' }