haraldwolff
/
collabora-online
mirror of https://github.com/CollaboraOnline/online.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(helm): add logging-operator flows 

Change-Id: I0bd8de7b4dfd974c1f9e0eeb24eefc229f70431f
Signed-off-by: genofire <geno+dev@fireorbit.de>
pull/7891/head
genofire 2023-10-04 14:15:08 +02:00 committed by Rashesh Padia
parent d07107953c
commit b282e64cb6
6 changed files with 328 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
kubernetes/helm/collabora-online/README.md
View File

@ -241,6 +241,47 @@ In order for Collaborative Editing and copy/paste to function correctly on kuber
---
## Kubernetes cluster logging
1. Install [Logging Operator](https://kube-logging.dev/) with an ClusterOutput "default".
2. Enable logging flow in your
`my_values.yaml`
```yaml
logging:
enabled: true
ecs: true
dedot: "-"
additionalFilters:
- grep:
exclude:
- key: "$['log']['level']"
pattern: '/(info|debug|trace)/'
globalOutputRefs:
- "default"
dynamicConfig:
logging:
enabled: true
ecs: true
dedot: "-"
globalOutputRefs:
- "default"
upload:
logging:
enabled: true
ecs: true
dedot: "-"
globalOutputRefs:
- "default"
```
* `dedot`: usefull if the Logging has an [global filter](https://kube-logging.dev/4.0/docs/configuration/crds/v1beta1/logging_types/#loggingspec-globalfilters) for dedot an correction for selector is possible.
* `ecs`: Therefore the fields are remapped to filter to the [ElasticCommonSchema](https://www.elastic.co/guide/en/ecs/current/index.html).
* `additionalFilters`: Add more filter of the logging-operator
## Dynamic/Remote configuration in kubernetes
For big setups, you may not want to restart every pod to modify WOPI

12
kubernetes/helm/collabora-online/templates/_helpers.tpl
View File

@ -50,6 +50,18 @@ app.kubernetes.io/name: {{ include "collabora-online.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Selector Log labels
*/}}
{{- define "collabora-online.selectorLogLabels" -}}
{{- if .Values.logging.dedot }}
app{{.Values.logging.dedot }}kubernetes{{.Values.logging.dedot }}io/name: {{ include "collabora-online.name" . }}
app{{.Values.logging.dedot }}kubernetes{{.Values.logging.dedot }}io/instance: {{ .Release.Name }}
{{- else }}
{{ include "collabora-online.selectorLabels" . }}
{{- end }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}

81
kubernetes/helm/collabora-online/templates/dynamicConfig/flow_dynconfig.yaml 100644
View File

@ -0,0 +1,81 @@
{{- with .Values.dynamicConfig.logging }}
{{- if and .enabled $.Values.dynamicConfig.enabled }}
---
apiVersion: logging.banzaicloud.io/v1beta1
kind: Flow
metadata:
name: {{ include "collabora-online.fullname" $ }}-dynconfig
spec:
match:
- select:
labels:
type: dynconfig
{{- include "collabora-online.selectorLogLabels" $ | nindent 10 }}
container_names:
- "{{ $.Chart.Name }}-dynconfig"
filters:
- parser:
hash_value_field: "nginx"
reserve_data: true
remove_key_name_field: true
parse:
type: "multi_format"
patterns:
- format: "regexp"
# for
expression: '^(?<remote>[^ ]*) -?(?<host>.*) -?(?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S*) ?(?<path>[^\"]*) (?<httpversion>HTTP\/[0-9\.]+)?" (?<code>[^ ]*) (?<size>[^ ]*) "-?(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?:(?<upstream_address_list>[^\"-]*)|-)"?$'
types: 'code:integer,size:integer,upstream_address_list:array'
time_key: "time"
time_format: "%d/%b/%Y:%H:%M:%S %z"
- format: "none"
{{- if .ecs }}
- record_transformer:
enable_ruby: true
records:
- event.module: "nginx"
destination.domain: '${ !(record["nginx"]["referer"].nil?) ? URI(record["nginx"]["referer"]).host : record["nginx"]["host"] }'
url.original: '${ record["nginx"]["referer"] }${ record["nginx"]["path"] }'
url.domain: '${ !(record["nginx"]["referer"].nil?) ? URI(record["nginx"]["referer"]).host : record["nginx"]["host"] }'
url.path: '${ record["nginx"]["path"] }'
http.version: '${ record["nginx"]["httpversion"] }'
source.ip: '${ record["nginx"]["remote"] }'
related.ip: '${ record["nginx"]["remote"] }'
http.request.method: '${ record["nginx"]["method"] }'
http.request.referrer: '${ record["nginx"]["referer"] }'
user.name: '${ record["nginx"]["user"] }'
related.user: '${ record["nginx"]["user"] }'
http.request.useragent: '${ record["nginx"]["agent"] }'
user_agent.original: '${ record["nginx"]["agent"] }'
http.response.status_code: '${ record["nginx"]["code"] }'
nginx.ingress_controller.upstream_address_list: '${ record["nginx"]["upstream_address_list"] }'
# TODO split ip and port
nginx.ingress_controller.upstream.address.merged: '${ [record["upstream_address_list"]].flatten&.last }'
http.response.body.bytes: '${ record["nginx"]["size"] }'
event.created: '${ time * 1000 }'
event.kind: "event"
event.category: "web"
event.type: "access"
event.outcome: '${ record["nginx"]["code"].to_i < 400 ? "success" : "failure" }'
# for dashboard
fileset.name: '${[ "ingress_controller", "access" ]}'
# on second regex -> error handling
message: '${ record["nginx"]["message"] }'
remove_keys: "$['nginx']['remote'],$['nginx']['host'],$['nginx']['user'],$['nginx']['method'],$['nginx']['path'],$['nginx']['httpversion'],$['nginx']['code'],$['nginx']['size'],$['nginx']['referer'],$['nginx']['agent'],$['nginx']['upstream_address_list'],$['nginx']['message']"
{{- end }}
{{- with .additionalFilters }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .globalOutputRefs }}
globalOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .localOutputRefs }}
localOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

55
kubernetes/helm/collabora-online/templates/dynamicConfig/flow_dynconfig_upload.yaml 100644
View File

@ -0,0 +1,55 @@
{{- with .Values.dynamicConfig.upload.logging }}
{{- if and .enabled $.Values.dynamicConfig.enabled $.Values.dynamicConfig.upload.enabled }}
---
apiVersion: logging.banzaicloud.io/v1beta1
kind: Flow
metadata:
name: {{ include "collabora-online.fullname" $ }}-dynconfig-upload
spec:
match:
- select:
labels:
type: dynconfig
{{- include "collabora-online.selectorLogLabels" $ | nindent 10 }}
container_names:
- "{{ $.Chart.Name }}-dynconfig-upload"
filters:
- parser:
hash_value_field: "simple-upload"
reserve_data: true
remove_key_name_field: true
parse:
type: "multi_format"
patterns:
- format: "regexp"
# for
# [::ffff:172.17.87.208] [400] Upload failed with: That request is not supported
expression: '^\[(?<ip>[a-f0-9\.:]+)\]\s\[(?<statuscode>\d+)\]\s(?<message>.*)'
types: 'statuscode:integer'
- format: "none"
{{- if .ecs }}
- record_transformer:
enable_ruby: true
records:
- message: '${ record["simple-upload"]["message"] }'
related.ip: '${ record["simple-upload"]["ip"] }'
http.response.status_code: '${ record["simple-upload"]["statuscode"] }'
remove_keys: "$['simple-upload']['message'],$['simple-upload']['ip'],$['simple-upload']['statuscode']"
{{- end }}
{{- with .additionalFilters }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .globalOutputRefs }}
globalOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .localOutputRefs }}
localOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}{{/* end-if .enabled ... */}}
{{- end }}{{/* end-with .dynamicConfig.upload.logging */}}

104
kubernetes/helm/collabora-online/templates/flow.yaml 100644
View File

@ -0,0 +1,104 @@
{{- with .Values.logging }}
{{- if .enabled }}
apiVersion: logging.banzaicloud.io/v1beta1
kind: Flow
metadata:
name: {{ include "collabora-online.fullname" $ }}
spec:
match:
- select:
labels:
type: main
{{- include "collabora-online.selectorLogLabels" $ | nindent 10 }}
container_names:
- "{{ $.Chart.Name }}"
filters:
- parser:
hash_value_field: "collabora"
remove_key_name_field: true
reserve_data: true
parse:
type: "multi_format"
patterns:
- format: "regexp"
# regex for wopihost and wopidoc
# wsd-00001-00092 2023-04-20 15:34:56.729798 +0000 [ docbroker_001 ] ERR Failed to add session to [https://integrator.example.org:443/collabora/lptFile123.odt] with URI [https://integrator.example.org/collabora/lptFile123.odt?access_token=1&access_token_ttl=0&permission=edit]: No acceptable WOPI hosts found matching the target host [integrator.example.org] in config.| wsd/DocumentBroker.cpp:2251
# wsd-00001-00092 2023-04-20 15:34:56.729826 +0000 [ docbroker_001 ] ERR Unauthorized Request while starting session on https://integrator.example.org:443/collabora/lptFile123.odt for socket #33. Terminating connection. Error: No acceptable WOPI hosts found matching the target host [integrator.example.org] in config.| wsd/COOLWSD.cpp:4727
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>.*(session (to|on) \[?(?<wopidoc>[^\s\]]*)\]?.*)No acceptable WOPI hosts found matching the target host \[(?<wopihost>.+)\] in config.)\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "regexp"
# regex for just wopihost
# wsd-00001-00092 2023-04-20 15:34:56.729615 +0000 [ docbroker_001 ] ERR No acceptable WOPI hosts found matching the target host [integrator.example.org] in config.| wsd/Storage.cpp:263
# wsd-00001-00092 2023-04-20 15:34:56.729748 +0000 [ docbroker_001 ] ERR loading document exception: No acceptable WOPI hosts found matching the target host [integrator.example.org] in config.| wsd/DocumentBroker.cpp:2289
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>.*No acceptable WOPI hosts found matching the target host \[(?<wopihost>.+)\] in config.)\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "regexp"
# regex for address denied with ip
# wsd-00001-00056 2023-04-18 16:00:20.954860 +0000 [ websrv_poll ] WRN convert-to: Requesting address is denied: 0.0.0.0| wsd/COOLWSD.cpp:3540
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>.*address is denied: )(?<ip>\d+\.\d+\.\d+\.\d+)\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "regexp"
# regex for response code with statuscode
# wsd-00001-00040 2023-04-16 10:03:08.379757 +0000 [ remoteconfig_poll ] ERR Remote config server has response status code: 503| wsd/COOLWSD.cpp:1163
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>.*has response status code: (?<statuscode>\d+))\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer,statuscode:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "regexp"
# by default parsed, this line
# Adding trusted WOPI host: [cloud\.example\.de].
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>Adding trusted WOPI host): \[(?<wopihost>[^\]]+)\]\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "regexp"
# fallback to default message regex
expression: '^(?<process>\w+)-(?<pid>\d+)-(?<thread>\d+)\s+(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{4})\s+\[\s+(?<compenent>\w+)\s+\]\s+(?<level>\w+)\s+(?<message>.+)\|\s(?<code>[^:]+):(?<codeline>[0-9]+)'
types: 'pid:integer,thread:integer,codeline:integer'
time_key: "time"
time_type: "string"
time_format: "%Y-%m-%d %H:%M:%S.%N %z"
- format: "none"
{{- if .ecs }}
- record_transformer:
enable_ruby: true
records:
- event.kind: "event"
message: '${ record["collabora"]["message"] }'
log.level: '${ record["collabora"]["level"].gsub("ERR", "error").gsub("WRN", "warn").gsub("INF","info").gsub("DBG","debug").gsub("TRC", "trace") }'
related.ip: '${ record["collabora"]["ip"] }'
client.ip: '${ record["collabora"]["ip"] }'
remove_keys: "$['collabora']['message'],$['collabora']['level'],$['collabora']['ip']"
{{- end }}
{{- with .additionalFilters }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .globalOutputRefs }}
globalOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .localOutputRefs }}
localOutputRefs:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

35
kubernetes/helm/collabora-online/values.yaml
View File

@ -94,6 +94,23 @@ grafana:
grafana_dashboard: "1"
annotations: {}
# Logging
# This HelmChart could also deploy Flow for the [Logging-Operator](https://kube-logging.github.io/docs/)
# Configuration is optimzed for deliever to elasticsearch
logging:
# -- Deploy Flow for logging-operator
enabled: false
# -- Enable record filter for record_modify to the [ElasticCommonSchema](https://www.elastic.co/guide/en/ecs/current/index.html)
ecs: false
# -- if an filter (here or global) for dedot is active - for disable set `null`
dedot:
# -- Add other filters to Flow
additionalFilters: []
# -- Flows localOutputRefs for use of Outputs
localOutputRefs: []
# -- Flows globalOutputRefs for use of ClusterOutputs
globalOutputRefs:
- "default"
podAnnotations: {}
@ -215,6 +232,15 @@ autoscaling:
dynamicConfig:
enabled: false
logging:
enabled: false
ecs: false
dedot:
additionalFilters: [ ]
localOutputRefs: [ ]
globalOutputRefs:
- "default"
image:
repository: nginx
tag: 1.25
@ -257,6 +283,15 @@ dynamicConfig:
# hosts:
# - chart-example.local
logging:
enabled: false
ecs: false
dedot:
additionalFilters: [ ]
localOutputRefs: [ ]
globalOutputRefs:
- "default"
containerPort: 80
probes: