fluentd.yml

apiVersion: v1 kind: ServiceAccount metadata: name: fluentd namespace: kube-system

apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: fluentd namespace: kube-system rules:

• apiGroups: [""] resources:
  • namespaces
  • pods verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: fluentd namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: fluentd subjects:

• kind: ServiceAccount name: fluentd namespace: kube-system

---

apiVersion: v1 kind: ConfigMap metadata: name: fluentd-config namespace: kube-system labels: k8s-app: fluentd-elasticsearch data: fluent.conf: | @include containers.conf @include systemd.conf <match fluent.**> # this tells fluentd to not output its log on stdout @type null </match> containers.conf: | # here we read the logs from Docker's containers and parse them <source> @type tail @id in_tail_container_logs @label @containers path /var/log/containers/*.log pos_file /var/log/fluentd-containers.log.pos tag * read_from_head true <parse> @type json time_format %Y-%m-%dT%H:%M:%S.%NZ </parse> </source> <label @containers> <filter **> @type kubernetes_metadata @id filter_kube_metadata </filter> <filter **> @type record_transformer @id filter_containers_stream_transformer <record> stream_name ${tag_parts[3]} </record> </filter> <match kube-system> @type null </match> <match fluentd-elasticsearch> @type null </match> <match ingress> @type null </match> <match **> @type cloudwatch_logs @id out_cloudwatch_logs_containers region "#{ENV.fetch('REGION')}" log_group_name "/eks/#{ENV.fetch('CLUSTER_NAME')}/containers" log_stream_name_key stream_name remove_log_stream_name_key true auto_create_stream true <buffer> flush_interval 5 chunk_limit_size 2m queued_chunks_limit_size 32 flush_thread_count 5 retry_forever true </buffer> </match> </label> systemd.conf: | <source> @type systemd @id in_systemd_kubelet @label @systemd filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] <entry> field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} field_map_strict true </entry> path /run/log/journal pos_file /var/log/fluentd-journald-kubelet.pos read_from_head true tag kubelet.service </source> <source> @type systemd @id in_systemd_kubeproxy @label @systemd filters [{ "_SYSTEMD_UNIT": "kubeproxy.service" }] <entry> field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} field_map_strict true </entry> path /run/log/journal pos_file /var/log/fluentd-journald-kubeproxy.pos read_from_head true tag kubeproxy.service </source> <source> @type systemd @id in_systemd_docker @label @systemd filters [{ "_SYSTEMD_UNIT": "docker.service" }] <entry> field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} field_map_strict true </entry> path /run/log/journal pos_file /var/log/fluentd-journald-docker.pos read_from_head true tag docker.service </source>

<label @systemd>
  <filter **>
    @type record_transformer
    @id filter_systemd_stream_transformer
    <record>
      stream_name ${tag}-${record["hostname"]}
    </record>
  </filter>
  <match **>
    @type cloudwatch_logs
    @id out_cloudwatch_logs_systemd
    region "#{ENV.fetch('REGION')}"
    log_group_name "/eks/#{ENV.fetch('CLUSTER_NAME')}/systemd"
    log_stream_name_key stream_name
    auto_create_stream true
    reload_on_failure false
    remove_log_stream_name_key true
    <buffer>
      flush_interval 5s
      chunk_limit_size 2m
      queued_chunks_limit_size 32
      retry_forever true
    </buffer>
  </match>
  <match pattern>
    @type s3
    aws_key_id 005117692424
    aws_sec_key XXXXX
    s3_bucket vEZB24t6HE5Gr7p6E3HVpwQ5Gan+0x5kY1q+hmY+
    s3_region eu-west-1
    path logs/
    <parse>
      @type json
    </parse>
    # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
    # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
    <buffer tag,time>
      @type file
      path /var/log/fluent/s3
      timekey 3600 # 1 hour partition
      timekey_wait 10m
      timekey_use_utc true # use utc
      chunk_limit_size 256m
    </buffer>
  </match>
</label>

---

apiVersion: apps/v1 kind: DaemonSet metadata: name: fluentd-elasticsearch namespace: kube-system labels: k8s-app: fluentd-elasticsearch spec: selector: matchLabels: k8s-app: fluentd-elasticsearch template: metadata: labels: k8s-app: fluentd-elasticsearch spec: serviceAccountName: fluentd terminationGracePeriodSeconds: 30 initContainers: - name: copy-fluentd-config image: busybox command: ['sh', '-c', 'cp /config-volume/..data/* /fluentd/etc'] volumeMounts: - name: config-volume mountPath: /config-volume - name: fluentdconf mountPath: /fluentd/etc

  containers:
  - name: fluentd-elasticsearch
    image: fluent/fluentd-kubernetes-daemonset:v1.11.1-debian-elasticsearch7-1.3
    env:
      - name: FLUENT_ELASTICSEARCH_HOST
        value: "ss-sssss.eu-west-1.es.amazonaws.com"
      - name: FLUENT_ELASTICSEARCH_PORT
        value: "443"
      - name: FLUENT_ELASTICSEARCH_SCHEME
        value: "https"
      - name: FLUENTD_SYSTEMD_CONF
        value: disable
      - name: FLUENT_ELASTICSEARCH_USER
        value: "xpto"
      - name: FLUENT_ELASTICSEARCH_PASSWORD
        value: "<[5ts("
    resources:
      limits:
        memory: 512Mi
      requests:
        cpu: 100m
        memory: 200Mi
    volumeMounts:
    - name: varlog
      mountPath: /var/log
    - name: varlibdockercontainers
      mountPath: /var/lib/docker/containers
      readOnly: true
  terminationGracePeriodSeconds: 30

  # containers:
  # - name: fluentd-cloudwatch
  #   image: fluent/fluentd-kubernetes-daemonset:v1.1-debian-cloudwatch
  #   env:
  #     - name: REGION
  #       value: eu-central-1
  #     - name: CLUSTER_NAME
  #       value: xxxx
  #   resources:
  #     limits:
  #       memory: 200Mi
  #     requests:
  #       cpu: 100m
  #       memory: 200Mi
  #   volumeMounts:
  #   - name: config-volume
  #     mountPath: /config-volume
  #   - name: fluentdconf
  #     mountPath: /fluentd/etc
  #   - name: varlog
  #     mountPath: /var/log
  #   - name: varlibdockercontainers
  #     mountPath: /var/lib/docker/containers
  #     readOnly: true
  #   - name: runlogjournal
  #     mountPath: /run/log/journal
  #     readOnly: true
  volumes:
  - name: config-volume
    configMap:
      name: fluentd-config
  - name: fluentdconf
    emptyDir: {}
  - name: varlog
    hostPath:
      path: /var/log
  - name: varlibdockercontainers
    hostPath:
      path: /var/lib/docker/containers
  - name: runlogjournal
    hostPath:
      path: /run/log/journal

===================== apiVersion: v1 kind: ServiceAccount metadata: name: fluentd namespace: kube-system

apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: fluentd namespace: kube-system rules:

• apiGroups: [''] resources:
  • namespaces
  • pods verbs: ['get', 'list', 'watch']

---

apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: fluentd namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: fluentd subjects:

• kind: ServiceAccount name: fluentd namespace: kube-system

---

apiVersion: v1 kind: ConfigMap metadata: name: fluentd-config namespace: kube-system labels: k8s-app: fluentd-elasticsearch data: fluent.conf: | <match fluent.**> @type null </match>

<match **nginx-ingress**>
  @type null
</match>

<match **kube-system**>
  @type null
</match>

<match kubernetes.var.log.containers.**fluentd**.log>
  @type null
</match>

<match kubernetes.var.log.containers.**kube-system**.log>
  @type null
</match>

@include containers.conf

containers.conf: | <source> @type tail @id in_tail_container_logs @label @containers path /var/log/containers/nexus*.log exclude_path ["/var/log/containers/fluentd*.log"] pos_file /var/log/fluentd-containers.log.pos tag * read_from_head true <parse> @type json time_format %Y-%m-%dT%H:%M:%S.%NZ </parse> <match nginx-ingress**> @type null </match> <match fluentd-elasticsearch**> @type null </match> </source>

<match nginx-ingress**>
  @type null
</match>

<match fluentd-elasticsearch**>
  @type null
</match>

<match cluster**>
  @type null
</match>

<match kubernetes.var.log.containers.**fluentd**.log>
  @type null
</match>

<match kubernetes.var.log.containers.**kube-system**.log>
  @type null
</match>

<match kubernetes.var.log.containers.**ingress**.log>
  @type null
</match>

<label @containers>
  <filter **>
    @type kubernetes_metadata
    @id filter_kube_metadata
  </filter>

  <filter **>
    @type record_transformer
    @id filter_containers_stream_transformer
    <record>
      stream_name ${tag_parts[3]}
    </record>
  </filter>

  <match **_kube-system_**>
    @type null
  </match>

  <match **nginx-ingress-controller**>
    @type null
  </match>

  <match **cluster-autoscaler**>
    @type null
  </match>

  <match **>
    @type cloudwatch_logs
    @id out_cloudwatch_logs_containers
    region "#{ENV.fetch('REGION')}"
    log_group_name "/eks/#{ENV.fetch('CLUSTER_NAME')}/containers"
    log_stream_name_key stream_name
    remove_log_stream_name_key true
    auto_create_stream true
    <buffer>
      flush_interval 5
      chunk_limit_size 2m
      queued_chunks_limit_size 32
      flush_thread_count 5
      retry_max_times 10
      retry_timeout 2
      retry_exponential_backoff_base 2
    </buffer>
  </match>
</label>

---

apiVersion: apps/v1 kind: DaemonSet metadata: name: fluentd-elasticsearch namespace: kube-system labels: k8s-app: fluentd-elasticsearch spec: selector: matchLabels: k8s-app: fluentd-elasticsearch template: metadata: labels: k8s-app: fluentd-elasticsearch spec: serviceAccountName: fluentd initContainers: - name: copy-fluentd-config image: busybox command: ['sh', '-c', 'cp /config-volume/..data/* /fluentd/etc'] volumeMounts: - name: config-volume mountPath: /config-volume - name: fluentdconf mountPath: /fluentd/etc

  containers:
    - name: fluentd-elasticsearch
      image: fluent/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1
      env:
        - name: FLUENT_ELASTICSEARCH_HOST
          value: 'XXXX'
        - name: FLUENT_ELASTICSEARCH_PORT
          value: '443'
        - name: FLUENT_ELASTICSEARCH_SCHEME
          value: 'https'
        - name: FLUENT_SYSTEMD_CONF
          value: disable
        - name: FLUENT_ELASTICSEARCH_USER
          value: 'nexus-sa'
        - name: FLUENT_ELASTICSEARCH_PASSWORD
          value: 'XXXXX'
      resources:
        limits:
          memory: 520Mi
        requests:
          cpu: 100m
          memory: 200Mi
      volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true

    - name: fluentd-cloudwatch
      image: fluent/fluentd-kubernetes-daemonset:v1.12.3-debian-cloudwatch-1.3
      env:
        - name: REGION
          value: eu-west-2
        - name: CLUSTER_NAME
          value: yyyyy
      resources:
        limits:
          memory: 200Mi
        requests:
          cpu: 100m
          memory: 200Mi
      volumeMounts:
        - name: config-volume
          mountPath: /config-volume
        - name: fluentdconf
          mountPath: /fluentd/etc
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
  terminationGracePeriodSeconds: 30
  volumes:
    - name: config-volume
      configMap:
        name: fluentd-config
    - name: fluentdconf
      emptyDir: {}
    - name: varlog
      hostPath:
        path: /var/log
    - name: varlibdockercontainers
      hostPath:
        path: /var/lib/docker/containers
    - name: runlogjournal
      hostPath:
        path: /run/log/journal