Epochly AI Inference: Configuration

from epochly.inference.safety.privacy import InputRedactor
redactor = InputRedactor(
    patterns=[
        r"\b\d{3}-\d{2}-\d{4}\b",           # SSN
        r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",  # Email
        r"\b\d{16}\b",                        # Credit card
    ],
    replacement="[REDACTED]",
)
clean_text = redactor.redact("SSN: 123-45-6789")
# "SSN: [REDACTED]"

from epochly.inference.safety.privacy import TenantIsolation
# From constructor
isolation = TenantIsolation(tenant_id="customer_abc")
# From environment (reads EPOCHLY_TENANT_ID)
isolation = TenantIsolation.from_env()
# Namespace a cache key
namespaced = isolation.namespace_key("model:bert:input_hash_abc123")
# "customer_abc:model:bert:input_hash_abc123"

from epochly.inference.safety.privacy import AuditLogger
audit = AuditLogger(max_entries=10_000)
audit.log_event(
    operation="canary_validation",
    model_id=42,
    optimization_name="torch_compile_bert",
    result="PASS",
    details={"cosine_sim": 0.998, "max_diff": 0.0012},
)
# Query recent entries
entries = audit.get_entries_since(time.time() - 3600)  # Last hour

from epochly.inference.cache_l2 import L2Cache, L2CacheConfig
l2 = L2Cache(L2CacheConfig(
    db_path="~/.epochly/inference_cache/cache.db",
    ttl_seconds=86400,
    encryption_key=b"32-byte-aes-256-key-here!!!!!!!",  # Optional
))

from epochly.inference.cache_l3 import L3Cache, L3CacheConfig
l3 = L3Cache(L3CacheConfig(
    redis_url="redis://localhost:6379/0",
    key_prefix="tenant_a:",
    ttl_seconds=86400,
    tls_enabled=False,
    socket_timeout=5.0,
))

from epochly.inference.serving.llm_companion import (
    LLMCompanionAdapter,
    LLMCompanionConfig,
)
companion = LLMCompanionAdapter(
    runtime_url="http://localhost:8000",
    config=LLMCompanionConfig(
        max_concurrent_requests=64,
        cache_enabled=True,
        cache_max_size=10_000,
        default_timeout_seconds=120.0,
        api_path="/v1/completions",
    ),
)

from epochly.inference.serving.ray_serve_wrapper import (
    EpochlyRayServeWrapper,
    RayServeConfig,
    epochly_serve,
)
# Via wrapper
wrapper = EpochlyRayServeWrapper(RayServeConfig(
    enable_telemetry=True,
    enable_priority_routing=True,
))
# Via decorator
@epochly_serve(config=RayServeConfig())
def predict(model, data):
    return model(data)

from epochly.inference.security import SecurityValidator
validator = SecurityValidator(max_input_size_bytes=10 * 1024 * 1024)
report = validator.run_all_checks(
    cache=inference_cache,
    privacy_mode="ephemeral",
    encryption_key_present=False,
    redaction_patterns_configured=False,
    config={"canary_enabled": True},
)

from epochly.inference.compilation.safety_monitor import TorchCompileSafetyMonitor
monitor = TorchCompileSafetyMonitor(
    max_graph_breaks=5,
    memory_growth_threshold_mb=500.0,
)
# Pre-compile check
result = monitor.pre_compile_check(model, sample_input)
# Post-compile health
health = monitor.check_health(pre_mb=1000, post_mb=1050)
# Output validity
validity = monitor.check_output_validity(model_output)

from fastapi import FastAPI
from fastapi.responses import PlainTextResponse
from epochly.inference.metrics.prometheus_exporter import PrometheusExporter
from epochly.inference.metrics.inference_metrics import InferenceMetrics
from epochly.inference.metrics.cost_estimator import CostEstimator
app = FastAPI()
inference_metrics = InferenceMetrics()
cost_estimator = CostEstimator(gpu_name="A100_80GB")
@app.get("/metrics")
def metrics():
    return PlainTextResponse(
        content=PrometheusExporter.export(
            inference_metrics,
            cost=cost_estimator,
            model_labels={
                id(model): {
                    "model_name": "bert-base",
                    "model_version": "v2.1",
                    "endpoint": "/predict",
                }
            },
        ),
        media_type="text/plain; version=0.0.4",
    )