Add practical examples to multiple files
- LF9-03 Virtualisierung: Docker Compose + Volume examples - LF6-02 Frontend: To-Do list practical example - LF8-04 ETL: Complete ETL pipeline example - LF6-04 Sicherheit: Express.js security headers - LF2-04 Nutzwertanalyse: Cloud provider selection example - LF9-04 Monitoring: Prometheus alerts + Python logging
This commit is contained in:
@@ -207,6 +207,81 @@ ETL - Fehlerstrategien
|
||||
|
||||
---
|
||||
|
||||
## Praktisches Beispiel: Vollständiger ETL-Pipeline
|
||||
|
||||
```python
|
||||
import pandas as pd
|
||||
import requests
|
||||
from sqlalchemy import create_engine
|
||||
import logging
|
||||
|
||||
# Logging konfigurieren
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def etl_pipeline():
|
||||
"""
|
||||
Vollständiger ETL-Pipeline für Verkaufsdaten
|
||||
"""
|
||||
|
||||
# === EXTRACT ===
|
||||
logger.info("Starte Extraktion...")
|
||||
|
||||
# Aus CSV
|
||||
kunden_df = pd.read_csv('daten/kunden.csv')
|
||||
bestellungen_df = pd.read_csv('daten/bestellungen.csv')
|
||||
|
||||
# Aus API
|
||||
try:
|
||||
response = requests.get('https://api.shop.de/produkte', timeout=30)
|
||||
produkte_df = pd.DataFrame(response.json())
|
||||
except Exception as e:
|
||||
logger.error(f"API Fehler: {e}")
|
||||
produkte_df = pd.DataFrame()
|
||||
|
||||
logger.info(f"Extrahiert: {len(kunden_df)} Kunden, {len(bestellungen_df)} Bestellungen")
|
||||
|
||||
# === TRANSFORM ===
|
||||
logger.info("Starte Transformation...")
|
||||
|
||||
# Daten bereinigen
|
||||
kunden_df = kunden_df.drop_duplicates()
|
||||
kunden_df['email'] = kunden_df['email'].str.lower().str.strip()
|
||||
kunden_df['erstellt_am'] = pd.to_datetime(kunden_df['erstellt_am'])
|
||||
|
||||
# Berechnungen
|
||||
bestellungen_df['umsatz_mit_mwst'] = bestellungen_df['umsatz_netto'] * 1.19
|
||||
|
||||
# JOIN: Bestellungen mit Kunden verbinden
|
||||
merged_df = bestellungen_df.merge(
|
||||
kunden_df[['kunden_id', 'name', 'stadt']],
|
||||
on='kunden_id',
|
||||
how='left'
|
||||
)
|
||||
|
||||
# Aggregation: Umsatz pro Stadt
|
||||
umsatz_pro_stadt = merged_df.groupby('stadt')['umsatz_netto'].sum().reset_index()
|
||||
|
||||
logger.info(f"Transformation abgeschlossen: {len(merged_df)} Datensätze")
|
||||
|
||||
# === LOAD ===
|
||||
logger.info("Starte Laden...")
|
||||
|
||||
# Datenbank-Verbindung
|
||||
engine = create_engine('postgresql://user:pass@localhost:5432/warehouse')
|
||||
|
||||
# In Datenbank laden
|
||||
merged_df.to_sql('fact_bestellungen', engine, if_exists='replace', index=False)
|
||||
umsatz_pro_stadt.to_sql('dim_umsatz_stadt', engine, if_exists='replace', index=False)
|
||||
|
||||
logger.info("ETL Pipeline erfolgreich abgeschlossen!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
etl_pipeline()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Querverweise
|
||||
|
||||
- [[LF8-03-Datenformate|Zurück: Datenformate]]
|
||||
|
||||
@@ -108,6 +108,59 @@ EXPOSE 3000
|
||||
CMD ["node", "server.js"]
|
||||
```
|
||||
|
||||
### Praktisches Beispiel: Docker Compose
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# Webanwendung
|
||||
app:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- DATABASE_URL=postgres://db:5432/webapp
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
|
||||
# Datenbank
|
||||
db:
|
||||
image: postgres:15-alpine
|
||||
volumes:
|
||||
- db_data:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=geheim
|
||||
- POSTGRES_DB=webapp
|
||||
|
||||
# Cache
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
|
||||
volumes:
|
||||
db_data:
|
||||
```
|
||||
|
||||
### Praktisches Beispiel: Docker Volume
|
||||
|
||||
```bash
|
||||
# Volume erstellen
|
||||
docker volume create mydata
|
||||
|
||||
# Volume einhängen
|
||||
docker run -v mydata:/data ubuntu
|
||||
|
||||
# Volumes auflisten
|
||||
docker volume ls
|
||||
|
||||
# Unbenutzte Volumes löschen
|
||||
docker volume prune
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Kubernetes
|
||||
|
||||
@@ -61,6 +61,73 @@ scrape_configs:
|
||||
- targets: ['localhost:9100']
|
||||
```
|
||||
|
||||
### Praktisches Beispiel: Alert-Regeln
|
||||
|
||||
```yaml
|
||||
# alerts.yml
|
||||
groups:
|
||||
- name: server_alerts
|
||||
rules:
|
||||
# Hohe CPU-Auslastung
|
||||
- alert: HighCPU
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Hohe CPU-Auslastung auf {{ $labels.instance }}"
|
||||
description: "CPU Auslastung ist seit 5 Minuten über 80%"
|
||||
|
||||
# Wenig Speicherplatz
|
||||
- alert: DiskSpaceLow
|
||||
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Wenig Speicherplatz auf {{ $labels.instance }}"
|
||||
|
||||
# Service ausgefallen
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.job }} Service ausgefallen"
|
||||
```
|
||||
|
||||
### Praktisches Beispiel: Python Logging
|
||||
|
||||
```python
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
|
||||
# Logger konfigurieren
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(sys.stdout),
|
||||
logging.handlers.RotatingFileHandler(
|
||||
'app.log',
|
||||
maxBytes=10_000_000, # 10 MB
|
||||
backupCount=5
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Log-Ebenen nutzen
|
||||
logger.debug("Detaillierte Debug-Info")
|
||||
logger.info("Anwendung gestartet")
|
||||
logger.warning("Warnung: Konfiguration fehlt")
|
||||
logger.error("Fehler: Datenbank nicht erreichbar")
|
||||
logger.critical("Kritisch: System muss heruntergefahren werden")
|
||||
```
|
||||
|
||||
### Grafana Dashboard
|
||||
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user