通过 Pub/Sub 接入 Google Cloud 日志

PROJECT_ID="my-project"
REGION="us-central1"
NETWORK="my-vpc"
SUBNET="cloud-run-subnet"
DATAKIT_IPV4="10.0.1.10"
DATAKIT_VM="datakit-vm"
DATAKIT_ZONE="us-central1-a"

gcloud services enable run.googleapis.com pubsub.googleapis.com \
  logging.googleapis.com cloudbuild.googleapis.com artifactregistry.googleapis.com \
  compute.googleapis.com iam.googleapis.com --project "$PROJECT_ID"

PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format='value(projectNumber)')

[http_api]
  listen = "10.0.1.10:9529"

SUBNET_CIDR=$(gcloud compute networks subnets describe "$SUBNET" \
  --project "$PROJECT_ID" --region "$REGION" --format='value(ipCidrRange)')
gcloud compute instances add-tags "$DATAKIT_VM" \
  --project "$PROJECT_ID" --zone "$DATAKIT_ZONE" --tags pubsub-datakit
gcloud compute firewall-rules create allow-pubsub-datakit \
  --project "$PROJECT_ID" --network "$NETWORK" --direction INGRESS \
  --action ALLOW --rules tcp:9529 --target-tags pubsub-datakit \
  --source-ranges "${SUBNET_CIDR:?Subnet CIDR is required}"

curl -i --fail --show-error --max-time 5 "http://10.0.1.10:9529/v1/ping"

gcloud iam service-accounts create forwarder-build --project "$PROJECT_ID"
gcloud iam service-accounts create forwarder-run --project "$PROJECT_ID"
gcloud iam service-accounts create forwarder-push --project "$PROJECT_ID"

gcloud projects add-iam-policy-binding "$PROJECT_ID" \
  --member "serviceAccount:forwarder-build@${PROJECT_ID}.iam.gserviceaccount.com" \
  --role roles/run.builder

git clone https://github.com/TrueWatchTech/google-cloud-log-forwarder.git
cd google-cloud-log-forwarder

cat > env.yaml <<EOF
DATAKIT_URL: "http://${DATAKIT_IPV4}:9529/v1/write/logging"
EXPECTED_PROJECT_ID: "${PROJECT_ID}"
EXPECTED_SUBSCRIPTION: "projects/${PROJECT_ID}/subscriptions/application-logs-push"
MEASUREMENT: "gcp_pubsub"
EOF

gcloud run deploy pubsub-log-forwarder \
  --source . --project "$PROJECT_ID" --region "$REGION" \
  --no-allow-unauthenticated \
  --build-service-account "projects/${PROJECT_ID}/serviceAccounts/forwarder-build@${PROJECT_ID}.iam.gserviceaccount.com" \
  --service-account "forwarder-run@${PROJECT_ID}.iam.gserviceaccount.com" \
  --network "$NETWORK" --subnet "$SUBNET" --vpc-egress private-ranges-only \
  --cpu 1 --memory 512Mi --min-instances 0 --max-instances 1 --concurrency 1 \
  --env-vars-file env.yaml

SERVICE_URL=$(gcloud run services describe pubsub-log-forwarder \
  --project "$PROJECT_ID" --region "$REGION" --format='value(status.url)')

gcloud run services add-iam-policy-binding pubsub-log-forwarder \
  --project "$PROJECT_ID" --region "$REGION" \
  --member "serviceAccount:forwarder-push@${PROJECT_ID}.iam.gserviceaccount.com" \
  --role roles/run.invoker

gcloud pubsub topics create application-logs --project "$PROJECT_ID"
gcloud beta services identity create \
  --service pubsub.googleapis.com --project "$PROJECT_ID"
gcloud iam service-accounts add-iam-policy-binding \
  "forwarder-push@${PROJECT_ID}.iam.gserviceaccount.com" \
  --project "$PROJECT_ID" \
  --member "serviceAccount:service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com" \
  --role roles/iam.serviceAccountTokenCreator

gcloud pubsub subscriptions create application-logs-push \
  --project "$PROJECT_ID" --topic application-logs --ack-deadline 60 \
  --push-endpoint "${SERVICE_URL}/pubsub" \
  --push-auth-service-account "forwarder-push@${PROJECT_ID}.iam.gserviceaccount.com" \
  --push-auth-token-audience "$SERVICE_URL"

gcloud logging sinks create application-logs \
  "pubsub.googleapis.com/projects/${PROJECT_ID}/topics/application-logs" \
  --project "$PROJECT_ID" \
  --log-filter="logName=\"projects/${PROJECT_ID}/logs/pubsub-guide-demo\" AND (jsonPayload:* OR textPayload:*)"

SINK_WRITER=$(gcloud logging sinks describe application-logs \
  --project "$PROJECT_ID" --format='value(writerIdentity)')

gcloud pubsub topics add-iam-policy-binding application-logs \
  --project "$PROJECT_ID" --member "$SINK_WRITER" --role roles/pubsub.publisher

TEST_ID="pubsub-$(date -u +%Y%m%dT%H%M%SZ)"
gcloud logging write pubsub-guide-demo \
  "{\"message\":\"${TEST_ID}-json\"}" \
  --payload-type=json --severity=INFO --project "$PROJECT_ID"
gcloud logging write pubsub-guide-demo "${TEST_ID}-text" \
  --payload-type=text --severity=INFO --project "$PROJECT_ID"

gcloud logging read \
  "logName=\"projects/${PROJECT_ID}/logs/pubsub-guide-demo\" AND SEARCH(\"${TEST_ID}\")" \
  --project "$PROJECT_ID" --freshness=1h --limit=10 --format=json
gcloud logging read \
  "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"pubsub-log-forwarder\" AND resource.labels.location=\"${REGION}\" AND jsonPayload.receipt:*" \
  --project "$PROJECT_ID" --freshness=1h --limit=20 --format=json

APP_FILTER=$(cat <<'EOF'
resource.type="k8s_container"
resource.labels.cluster_name="my-cluster"
resource.labels.namespace_name="my-namespace"
resource.labels.container_name="my-app"
(log_id("stdout") OR log_id("stderr"))
(jsonPayload:* OR textPayload:*)
EOF
)
gcloud logging read "$APP_FILTER" \
  --project "$PROJECT_ID" --freshness=1h --limit=5 --format=json

gcloud logging sinks update application-logs \
  --project "$PROJECT_ID" --log-filter="$APP_FILTER"