Kubernetes アプリケーションにおける RUM-APM-LOG の連携分析¶

apiVersion: v1
kind: Namespace
metadata:
  name: datakit
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: datakit
rules:
  - apiGroups:
      - rbac.authorization.k8s.io
    resources:
      - clusterroles
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
      - nodes/proxy
      - namespaces
      - pods
      - pods/log
      - events
      - services
      - endpoints
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - apps
    resources:
      - deployments
      - daemonsets
      - statefulsets
      - replicasets
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - batch
    resources:
      - jobs
      - cronjobs
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - guance.com
    resources:
      - datakits
    verbs:
      - get
      - list
  - apiGroups:
      - metrics.k8s.io
    resources:
      - pods
      - nodes
    verbs:
      - get
      - list
  - nonResourceURLs: ["/metrics"]
    verbs: ["get"]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: datakit
  namespace: datakit

---
apiVersion: v1
kind: Service
metadata:
  name: datakit-service
  namespace: datakit
spec:
  selector:
    app: daemonset-datakit
  ports:
    - protocol: TCP
      port: 9529
      targetPort: 9529

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: datakit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: datakit
subjects:
  - kind: ServiceAccount
    name: datakit
    namespace: datakit

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: daemonset-datakit
  name: datakit
  namespace: datakit
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: daemonset-datakit
  template:
    metadata:
      labels:
        app: daemonset-datakit
    spec:
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - env:
        - name: HOST_IP
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: ENV_K8S_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: ENV_DATAWAY
          value: https://openway.guance.com?token=XXXXXX
        - name: ENV_GLOBAL_HOST_TAGS # 非選出タグ
          value: host=__datakit_hostname,host_ip=__datakit_ip,cluster_name_k8s=k8s-prod
        - name: ENV_DEFAULT_ENABLED_INPUTS
          value: cpu,disk,diskio,mem,swap,system,hostobject,net,host_processes,container,statsd,ebpf,rum
        - name: ENV_ENABLE_ELECTION
          value: enable
        - name: ENV_GLOBAL_ENV_TAGS # 選出タグにのみ有効
          value: cluster_name_k8s=k8s-prod
        - name: ENV_HTTP_LISTEN
          value: 0.0.0.0:9529
        - name: ENV_NAMESPACE # 選出用
          value: guance-k8s-demo
        #- name: ENV_LOG_LEVEL
        #  value: debug
        #- name: ENV_K8S_CLUSTER_NAME
        #  value: k8s-prod
        image: pubrepo.guance.com/datakit/datakit:1.4.10
        imagePullPolicy: Always
        name: datakit
        ports:
        - containerPort: 9529
          hostPort: 9529
          name: port
          protocol: TCP
        securityContext:
          privileged: true
        volumeMounts:
        - mountPath: /var/run
          name: run
        - mountPath: /var/lib
          name: lib
        - mountPath: /var/log
          name: log
        #- mountPath: /var/run/containerd/containerd.sock
        #  name: containerd-socket
        #  readOnly: true
        - mountPath: /usr/local/datakit/conf.d/container/container.conf
          name: datakit-conf
          subPath: container.conf
        - mountPath: /usr/local/datakit/conf.d/log/logfwdserver.conf
          name: datakit-conf
          subPath: logfwdserver.conf
        - mountPath: /usr/local/datakit/conf.d/ddtrace/ddtrace.conf
          name: datakit-conf
          subPath: ddtrace.conf
        - mountPath: /host/proc
          name: proc
          readOnly: true
        - mountPath: /host/dev
          name: dev
          readOnly: true
        - mountPath: /host/sys
          name: sys
          readOnly: true
        - mountPath: /rootfs
          name: rootfs
        - mountPath: /sys/kernel/debug
          name: debugfs

        workingDir: /usr/local/datakit
      hostIPC: true
      hostPID: true
      restartPolicy: Always
      serviceAccount: datakit
      serviceAccountName: datakit
      tolerations:
        - operator: Exists
      volumes:
        - configMap:
            name: datakit-conf
          name: datakit-conf
        - hostPath:
            path: /var/run
          name: run
        - hostPath:
            path: /var/lib
          name: lib
        - hostPath:
            path: /var/log
          name: log

        - hostPath:
            path: /proc
            type: ""
          name: proc
        - hostPath:
            path: /dev
            type: ""
          name: dev
        - hostPath:
            path: /sys
            type: ""
          name: sys
        - hostPath:
            path: /
            type: ""
          name: rootfs
        - hostPath:
            path: /sys/kernel/debug
            type: ""
          name: debugfs
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: datakit-conf
  namespace: datakit
data:
  #### container
  container.conf: |-
    [inputs.container]
      docker_endpoint = "unix:///var/run/docker.sock"
      containerd_address = "/var/run/containerd/containerd.sock"

      enable_container_metric = true
      enable_k8s_metric = true
      enable_pod_metric = true

      ## Containers logs to include and exclude, default collect all containers. Globs accepted.
      container_include_log = []
      container_exclude_log = ["image:*"]
      #container_exclude_log = ["image:pubrepo.guance.com/datakit/logfwd*", "image:pubrepo.guance.com/datakit/datakit*"]

      exclude_pause_container = true

      ## Removes ANSI escape codes from text strings
      logging_remove_ansi_escape_codes = false

      kubernetes_url = "https://kubernetes.default:443"

      ## Authorization level:
      ##   bearer_token -> bearer_token_string -> TLS
      ## Use bearer token for authorization. ('bearer_token' takes priority)
      ## linux at:   /run/secrets/kubernetes.io/serviceaccount/token
      ## windows at: C:\var\run\secrets\kubernetes.io\serviceaccount\token
      bearer_token = "/run/secrets/kubernetes.io/serviceaccount/token"
      # bearer_token_string = "<your-token-string>"

      [inputs.container.tags]
        # some_tag = "some_value"
        # more_tag = "some_other_value"

  #### ddtrace
  ddtrace.conf: |-
    [[inputs.ddtrace]]
      endpoints = ["/v0.3/traces", "/v0.4/traces", "/v0.5/traces"]
      # ignore_resources = []
      customer_tags = ["node_ip"]
      [inputs.ddtrace.close_resource]
        "*" = ["PUT /nacos/*","GET /nacos/*","POST /nacos/*"]
      ## tags is ddtrace configed key value pairs
      # [inputs.ddtrace.tags]
        # some_tag = "some_value"
        # more_tag = "some_other_value"

  #### logfwdserver
  logfwdserver.conf: |-
    [inputs.logfwdserver]
      ## logfwd 受信側のリスニングアドレスとポート
      address = "0.0.0.0:9531"

      [inputs.logfwdserver.tags]
      # some_tag = "some_value"
      # more_tag = "some_other_value"

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile        on;
    #tcp_nopush     on;

    client_max_body_size  50m;
    #keepalive_timeout  0;
    keepalive_timeout  65;
    #gzip  on;
    server {
        listen       80;
        server_name  localhost;

        location / {
            root  /data/nginx/web/dist;
            index  index.html index.htm;
            try_files $uri $uri/ /index.html;
        }

            location /prod-api/{
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header REMOTE-HOST $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_pass http://172.16.0.229:30001/;
        }

        location /nginx_status{
            stub_status;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

apiVersion: v1
kind: Service
metadata:
  name: web-service
  labels:
    app: web-service
spec:
  selector:
    app: web-service
  ports:
    - protocol: TCP
      port: 80
      nodePort: 30000
      targetPort: 80
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-service
  labels:
    app: web-service
spec:
  replicas: 1
  selector:
    matchLabels:
      app: web-service
  template:
    metadata:
      labels:
        app: web-service
    spec:
      containers:
      - env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name

        name: web-service
        image: 47.96.6.150:5000/df-demo/demo-web:v1
        #command: ["sh","-c"]
        ports:
        - containerPort: 80
          protocol: TCP

apiVersion: v1
kind: Service
metadata:
  name: gateway-service
  labels:
    app: gateway-service
spec:
  selector:
    app: gateway-service
  ports:
    - protocol: TCP
      port: 9299
      nodePort: 30001
      targetPort: 9299
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gateway-service
  labels:
    app: gateway-service
spec:
  replicas: 2
  selector:
    matchLabels:
      app: gateway-service
  template:
    metadata:
      labels:
        app: gateway-service
    spec:
      containers:
      - env:
        - name: DD_AGENT_HOST
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: NACOS_IP
          value: "172.16.0.230"
        - name: JAVA_OPTS
          value: |-
            -javaagent:/usr/dd-java-agent/agent/dd-java-agent.jar -Ddd.service.name=demo-k8s-gateway  -Ddd.tags=container_host:$(POD_NAME),node_ip:$(DD_AGENT_HOST) -Ddd.service.mapping=redis:redisk8s -Ddd.env=dev -Ddd.agent.port=9529
        - name: PARAMS
          value: "--spring.redis.host=$(NACOS_IP) --spring.nacos.ip=$(NACOS_IP)"
        name: gateway-service
        image: 47.96.6.150:5000/df-demo/demo-gateway:v1
        #command: ["sh","-c"]
        ports:
        - containerPort: 9299
          protocol: TCP
        volumeMounts:
        - mountPath: /usr/dd-java-agent/agent
          name: ddagent
      initContainers:
      - command:
        - sh
        - -c
        - set -ex;mkdir -p /ddtrace/agent;cp -r /datadog-init/* /ddtrace/agent;
        image: pubrepo.guance.com/datakit-operator/dd-lib-java-init
        imagePullPolicy: Always
        name: ddtrace-agent-sidecar
        volumeMounts:
        - mountPath: /ddtrace/agent
          name: ddagent
      restartPolicy: Always
      volumes:
      - emptyDir: {}
        name: ddagent

apiVersion: v1
kind: Service
metadata:
  name: auth-service
  labels:
    app: auth-service
spec:
  selector:
    app: auth-service
  ports:
    - protocol: TCP
      port: 9200
      targetPort: 9200
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: auth-service
  labels:
    app: auth-service
spec:
  replicas: 2
  selector:
    matchLabels:
      app: auth-service
  template:
    metadata:
      labels:
        app: auth-service
    spec:
      containers:
      - env:
        - name: DD_AGENT_HOST
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: NACOS_IP
          value: "172.16.0.230"
        - name: JAVA_OPTS
          value: |-
            -javaagent:/usr/dd-java-agent/agent/dd-java-agent.jar -Ddd.service.name=demo-k8s-auth  -Ddd.tags=container_host:$(POD_NAME),node_ip:$(DD_AGENT_HOST) -Ddd.service.mapping=redis:redisk8s -Ddd.env=dev -Ddd.agent.port=9529 
        - name: PARAMS
          value: "--spring.redis.host=$(NACOS_IP) --spring.nacos.ip=$(NACOS_IP)"
        name: auth-service
        image: 47.96.6.150:5000/df-demo/demo-auth:v1
        #command: ["sh","-c"]
        ports:
        - containerPort: 9200
          protocol: TCP
        volumeMounts:
        - mountPath: /usr/dd-java-agent/agent
          name: ddagent
      initContainers:
      - command:
        - sh
        - -c
        - set -ex;mkdir -p /ddtrace/agent;cp -r /datadog-init/* /ddtrace/agent;
        image: pubrepo.guance.com/datakit-operator/dd-lib-java-init
        imagePullPolicy: Always
        name: ddtrace-agent-sidecar
        volumeMounts:
        - mountPath: /ddtrace/agent
          name: ddagent
      restartPolicy: Always
      volumes:
      - emptyDir: {}
        name: ddagent

apiVersion: v1
kind: Service
metadata:
  name: system-service
spec:
  selector:
    app: system-pod
  ports:
    - protocol: TCP
      port: 9201
      #nodePort: 30001
      targetPort: 9201
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: system-deployment
  #labels:
  #  app: system-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: system-pod
  template:
    metadata:
      labels:
        app: system-pod
    spec:          
      containers:
      - name: system-container      
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: DD_AGENT_HOST
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: NACOS_IP
          value: "172.16.0.229"
        - name: DB_IP
          value: "172.16.0.230"
        - name: JAVA_OPTS
          value: |-
            -javaagent:/usr/dd-java-agent/agent/dd-java-agent.jar -Ddd.service.name=demo-k8s-system  -Ddd.tags=container_host:$(POD_NAME),node_ip:$(DD_AGENT_HOST) -Ddd.service.mapping=mysql:mysql-k8s,redis:redisk8s -Ddd.env=dev -Ddd.agent.port=9529 
        - name: PARAMS
          value: "--spring.redis.host=$(DB_IP) --spring.nacos.ip=$(NACOS_IP) --spring.db.ip=$(DB_IP)"
        image: 172.16.0.238/df-ruoyi/demo-system:v1
        #command: ["sh","-c"]
        ports:
        - containerPort: 9201
          protocol: TCP
        volumeMounts:
        - name: ddagent
          mountPath: /usr/dd-java-agent/agent
        - name: varlog
          mountPath: /data/app/logs/ruoyi-system
        resources:
          limits: 
            memory: 512Mi
          requests:
            memory: 256Mi
      - name: logfwd
        image: pubrepo.guance.com/datakit/logfwd:1.2.7
        env:
        - name: LOGFWD_DATAKIT_HOST
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.hostIP
        - name: LOGFWD_DATAKIT_PORT
          value: "9531"
        - name: LOGFWD_LOGFWD_ANNOTATION_DATAKIT_LOG_CONFIGS
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.annotations['datakit/log']
        - name: LOGFWD_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: LOGFWD_POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace  
        volumeMounts:
        - mountPath: /var/log
          name: varlog 
        - mountPath: /opt/logfwd/config
          name: logfwd-config
          subPath: config               
      initContainers:
      - name: ddtrace-agent-sidecar
        command:
        - sh
        - -c
        - set -ex;mkdir -p /ddtrace/agent;cp -r /datadog-init/* /ddtrace/agent;
        image: pubrepo.guance.com/datakit-operator/dd-lib-java-init
        imagePullPolicy: Always
        volumeMounts:
        - mountPath: /ddtrace/agent
          name: ddagent
      restartPolicy: Always
      volumes:
      - name: varlog
        emptyDir: {} 
      - name: ddagent
        emptyDir: {} 
      - configMap:
          name: logfwd-conf
        name: logfwd-config 

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: logfwd-conf
data:
  config: |
    [
        {            
            "loggings": [
                {
                    "logfiles": ["/var/log/info.log","/var/log/error.log"],
                    "source": "k8s-log-system",                   
                    "multiline_match": "^\\d{4}-\\d{2}-\\d{2}"
                }
            ]
        }
    ]