ingest.sh (1751B)
1 #!/bin/sh 2 # ============================================================ 3 # Data Ingestion Script 4 # ============================================================ 5 # Loads data into PostgreSQL database 6 7 set -e 8 9 echo "=========================================" 10 echo "Starting data ingestion..." 11 echo "=========================================" 12 13 # Database connection parameters 14 DB_HOST="${DB_HOST:-postgres}" 15 DB_PORT="${DB_PORT:-5432}" 16 DB_NAME="${DB_NAME:-opentel_db}" 17 DB_USER="${DB_USER:-opentel_user}" 18 DB_PASSWORD="${DB_PASSWORD:-opentel_pass}" 19 WAIT_TIME="${WAIT_TIME:-10}" 20 21 export PGPASSWORD="$DB_PASSWORD" 22 23 echo "Waiting for database to be ready..." 24 until pg_isready -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME"; do 25 echo "Database is unavailable - sleeping" 26 sleep 2 27 done 28 29 echo "Database is ready!" 30 echo "" 31 32 # Wait for products service to complete migrations 33 echo "Waiting ${WAIT_TIME}s for products service migrations to complete..." 34 sleep "$WAIT_TIME" 35 echo "" 36 37 # Load data files in order 38 echo "Loading products test data..." 39 if [ -f /data/products_test_data.sql ]; then 40 psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f /data/products_test_data.sql 41 echo "✓ Products test data loaded successfully" 42 else 43 echo "⚠ products_test_data.sql not found, skipping..." 44 fi 45 46 echo "" 47 echo "Backfilling inventory schema from product catalog..." 48 if [ -f /data/inventory_seed.sql ]; then 49 psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f /data/inventory_seed.sql 50 echo "✓ Inventory seed loaded successfully" 51 else 52 echo "⚠ inventory_seed.sql not found, skipping..." 53 fi 54 55 echo "" 56 echo "=========================================" 57 echo "Data ingestion completed!" 58 echo "========================================="