diff --git a/CHANGELOG.md b/CHANGELOG.md index daecb08..e0f4b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ All notable changes to this project will be documented in this file. - Bump testing-tools to `0.3.0-stackable0.0.0-dev` ([#91]). +### Fixed + +- Log file rollover fixed ([#107]). + [#55]: https://github.com/stackabletech/opensearch-operator/pull/55 [#63]: https://github.com/stackabletech/opensearch-operator/pull/63 [#76]: https://github.com/stackabletech/opensearch-operator/pull/76 @@ -38,6 +42,7 @@ All notable changes to this project will be documented in this file. [#94]: https://github.com/stackabletech/opensearch-operator/pull/94 [#97]: https://github.com/stackabletech/opensearch-operator/pull/97 [#100]: https://github.com/stackabletech/opensearch-operator/pull/100 +[#107]: https://github.com/stackabletech/opensearch-operator/pull/107 ## [25.11.0] - 2025-11-07 diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 8238db3..ed4674a 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -13,6 +13,7 @@ use crate::{ crd::v1alpha1, framework::{ builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::STACKABLE_LOG_DIR, role_group_utils, types::{kubernetes::ServiceName, operator::RoleGroupName}, }, @@ -23,96 +24,101 @@ pub const CONFIGURATION_FILE_OPENSEARCH_YML: &str = "opensearch.yml"; /// The cluster name. /// Type: string -pub const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name"; +const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name"; /// The list of hosts that perform discovery when a node is started. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_DISCOVERY_SEED_HOSTS: &str = "discovery.seed_hosts"; +const CONFIG_OPTION_DISCOVERY_SEED_HOSTS: &str = "discovery.seed_hosts"; /// By default, OpenSearch forms a multi-node cluster. Set `discovery.type` to `single-node` to /// form a single-node cluster. /// Type: string -pub const CONFIG_OPTION_DISCOVERY_TYPE: &str = "discovery.type"; +const CONFIG_OPTION_DISCOVERY_TYPE: &str = "discovery.type"; /// Specifies an address or addresses that an OpenSearch node publishes to other nodes for HTTP /// communication. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_HTTP_PUBLISH_HOST: &str = "http.publish_host"; +const CONFIG_OPTION_HTTP_PUBLISH_HOST: &str = "http.publish_host"; /// A list of cluster-manager-eligible nodes used to bootstrap the cluster. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES: &str = - "cluster.initial_cluster_manager_nodes"; +const CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES: &str = "cluster.initial_cluster_manager_nodes"; /// Binds an OpenSearch node to an address. /// Type: string -pub const CONFIG_OPTION_NETWORK_HOST: &str = "network.host"; +const CONFIG_OPTION_NETWORK_HOST: &str = "network.host"; /// Specifies an address or addresses that an OpenSearch node publishes to other nodes in the /// cluster so that they can connect to it. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_NETWORK_PUBLISH_HOST: &str = "network.publish_host"; +const CONFIG_OPTION_NETWORK_PUBLISH_HOST: &str = "network.publish_host"; /// The custom node attribute "role-group" /// Type: string -pub const CONFIG_OPTION_NODE_ATTR_ROLE_GROUP: &str = "node.attr.role-group"; +const CONFIG_OPTION_NODE_ATTR_ROLE_GROUP: &str = "node.attr.role-group"; /// A descriptive name for the node. /// Type: string -pub const CONFIG_OPTION_NODE_NAME: &str = "node.name"; +const CONFIG_OPTION_NODE_NAME: &str = "node.name"; /// Defines one or more roles for an OpenSearch node. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_NODE_ROLES: &str = "node.roles"; +const CONFIG_OPTION_NODE_ROLES: &str = "node.roles"; + +/// Defines the path for the logs +/// OpenSearch grants the required access rights, see +/// https://github.com/opensearch-project/OpenSearch/blob/3.4.0/server/src/main/java/org/opensearch/bootstrap/Security.java#L369 +/// The permissions "write" and "delete" are required for the log file rollover. +/// Type: string +const CONFIG_OPTION_PATH_LOGS: &str = "path.logs"; /// Specifies a list of distinguished names (DNs) that denote the other nodes in the cluster. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn"; +const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn"; /// Whether to enable TLS on the REST layer. If enabled, only HTTPS is allowed. /// Type: boolean -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str = - "plugins.security.ssl.http.enabled"; +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled"; /// Path to the cert PEM file used for TLS on the HTTP PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMCERT_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMCERT_FILEPATH: &str = "plugins.security.ssl.http.pemcert_filepath"; /// Path to the key PEM file used for TLS on the HTTP PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMKEY_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMKEY_FILEPATH: &str = "plugins.security.ssl.http.pemkey_filepath"; /// Path to the trusted CAs PEM file used for TLS on the HTTP PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_PEMTRUSTEDCAS_FILEPATH: &str = "plugins.security.ssl.http.pemtrustedcas_filepath"; /// Whether to enable TLS on internal node-to-node communication using the transport port. /// type: boolean -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_ENABLED: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_ENABLED: &str = "plugins.security.ssl.transport.enabled"; /// Path to the cert PEM file used for TLS on the transport PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMCERT_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMCERT_FILEPATH: &str = "plugins.security.ssl.transport.pemcert_filepath"; /// Path to the key PEM file used for TLS on the transport PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMKEY_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMKEY_FILEPATH: &str = "plugins.security.ssl.transport.pemkey_filepath"; /// Path to the trusted CAs PEM file used for TLS on the transport PORT. /// type: string -pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH: &str = +const CONFIG_OPTION_PLUGINS_SECURITY_SSL_TRANSPORT_PEMTRUSTEDCAS_FILEPATH: &str = "plugins.security.ssl.transport.pemtrustedcas_filepath"; /// Specifies an address or addresses that an OpenSearch node publishes to other nodes for /// transport communication. /// Type: (comma-separated) list of strings -pub const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host"; +const CONFIG_OPTION_TRANSPORT_PUBLISH_HOST: &str = "transport.publish_host"; const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; @@ -203,6 +209,13 @@ impl NodeConfig { CONFIG_OPTION_NODE_ATTR_ROLE_GROUP.to_owned(), json!(self.role_group_name), ); + config.insert( + CONFIG_OPTION_PATH_LOGS.to_owned(), + json!(format!( + "{STACKABLE_LOG_DIR}/{container}", + container = v1alpha1::Container::OpenSearch.to_container_name() + )), + ); config } @@ -616,6 +629,7 @@ mod tests { "discovery.type: \"zen\"\n", "network.host: \"0.0.0.0\"\n", "node.attr.role-group: \"data\"\n", + "path.logs: \"/stackable/log/opensearch\"\n", "plugins.security.nodes_dn: [\"CN=generated certificate for pod\"]\n", "plugins.security.ssl.http.enabled: true\n", "plugins.security.ssl.http.pemcert_filepath: \"/stackable/opensearch/config/tls/server/tls.crt\"\n", diff --git a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 index b2a0805..a2ebd69 100644 --- a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 @@ -13,6 +13,34 @@ data: appender.FILE.layout.type = OpenSearchJsonLayout appender.FILE.layout.type_name = server --- +# Expose the API of the Vector agent +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-api-config +data: + vector-api-config.yaml: | + api: + address: 0.0.0.0:8686 + enabled: true +--- +# Expose the API of the Vector agent of the pod opensearch-nodes-automatic-0 +apiVersion: v1 +kind: Service +metadata: + name: opensearch-nodes-automatic-vector +spec: + ports: + - name: vector + port: 8686 + protocol: TCP + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: automatic + type: ClusterIP +--- apiVersion: opensearch.stackable.tech/v1alpha1 kind: OpenSearchCluster metadata: @@ -37,10 +65,10 @@ spec: console: level: INFO file: - level: INFO + level: TRACE loggers: ROOT: - level: INFO + level: TRACE vector: console: level: INFO @@ -78,10 +106,26 @@ spec: - name: security-config mountPath: /stackable/opensearch/config/opensearch-security readOnly: true + - name: vector + env: + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml,/stackable/config/vector-api-config.yaml + ports: + - name: vector + containerPort: 8686 + protocol: TCP + volumeMounts: + - name: vector-api-config + mountPath: /stackable/config/vector-api-config.yaml + readOnly: true + subPath: vector-api-config.yaml volumes: - name: security-config secret: secretName: opensearch-security-config + - name: vector-api-config + configMap: + name: vector-api-config --- apiVersion: v1 kind: Secret diff --git a/tests/templates/kuttl/logging/30-test-opensearch.yaml b/tests/templates/kuttl/logging/30-test-opensearch.yaml index 6524f55..342af13 100644 --- a/tests/templates/kuttl/logging/30-test-opensearch.yaml +++ b/tests/templates/kuttl/logging/30-test-opensearch.yaml @@ -37,6 +37,7 @@ spec: securityContext: fsGroup: 1000 restartPolicy: OnFailure + backoffLimit: 10 --- apiVersion: v1 kind: ConfigMap @@ -88,6 +89,50 @@ data: f'No events were sent in "{componentId}".' + def check_log_file_rollover(): + response = requests.post( + 'http://opensearch-nodes-automatic-vector:8686/graphql', + json={ + 'query': """ + { + sources { + nodes { + componentId + metrics { + receivedBytesTotal { + receivedBytesTotal + } + } + } + } + } + """ + } + ) + + assert response.status_code == 200, \ + 'Cannot access the API of the vector agent.' + + result = response.json() + + sources = result['data']['sources']['nodes'] + for source in sources: + receivedBytes = source['metrics']['receivedBytesTotal'] + componentId = source['componentId'] + + if componentId == 'files_opensearch_server': + assert receivedBytes is not None + receivedBytes = receivedBytes['receivedBytesTotal'] + MAX_LOG_FILE_SIZE = 5_500_000 + expectedBytes = 2 * MAX_LOG_FILE_SIZE + assert receivedBytes >= expectedBytes, \ + 'Log file rollover did not yet happen twice ' \ + f'({receivedBytes:,.0f} Bytes of {expectedBytes:,d} Bytes received). ' \ + 'The first rollover requires write permission to rename the log file, ' \ + 'the second rollover additionally requires delete permission to remove the old log file.' + + if __name__ == '__main__': check_sent_events() + check_log_file_rollover() print('Test successful!') diff --git a/tests/templates/kuttl/metrics/01-rbac.yaml b/tests/templates/kuttl/metrics/01-rbac.yaml index d1a7ebc..f8f1537 100644 --- a/tests/templates/kuttl/metrics/01-rbac.yaml +++ b/tests/templates/kuttl/metrics/01-rbac.yaml @@ -1,37 +1,18 @@ --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: test-service-account ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-role -rules: - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - resourceNames: - - privileged - verbs: - - use ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-role-binding -subjects: - - kind: ServiceAccount - name: test-service-account - - kind: ServiceAccount - name: prometheus-stack-kube-prom-admission - - kind: ServiceAccount - name: prometheus-stack-kube-prom-operator - - kind: ServiceAccount - name: prometheus-stack-kube-prom-prometheus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: test-role +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + PROMETHEUS_HELM_RELEASE_NAME=prometheus-$NAMESPACE + PROMETHEUS_HELM_CHART_NAME=kube-prometheus-stack + + # `kube-prometheus-stack.fullname` in the Prometheus Helm Chart is used to create the + # ServiceAccount names. It is set to `<.Release.Name>-<.Chart.Name>` and truncated to + # 26 characters without dashes at the end, see + # https://github.com/prometheus-community/helm-charts/blob/kube-prometheus-stack-81.2.2/charts/kube-prometheus-stack/templates/_helpers.tpl#L22 + KUBE_PROMETHEUS_STACK_FULLNAME=$PROMETHEUS_HELM_RELEASE_NAME-$PROMETHEUS_HELM_CHART_NAME + KUBE_PROMETHEUS_STACK_FULLNAME=$(echo -n $KUBE_PROMETHEUS_STACK_FULLNAME | head --bytes=26) + export KUBE_PROMETHEUS_STACK_FULLNAME=$(echo -n $KUBE_PROMETHEUS_STACK_FULLNAME | sed 's/-*$//') + + envsubst '$KUBE_PROMETHEUS_STACK_FULLNAME' < 01_rbac.yaml | \ + kubectl apply --namespace=$NAMESPACE --filename=- diff --git a/tests/templates/kuttl/metrics/01_rbac.yaml b/tests/templates/kuttl/metrics/01_rbac.yaml new file mode 100644 index 0000000..19ed4db --- /dev/null +++ b/tests/templates/kuttl/metrics/01_rbac.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account + - kind: ServiceAccount + name: $KUBE_PROMETHEUS_STACK_FULLNAME-admission + - kind: ServiceAccount + name: $KUBE_PROMETHEUS_STACK_FULLNAME-operator + - kind: ServiceAccount + name: $KUBE_PROMETHEUS_STACK_FULLNAME-prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/metrics/10-install-prometheus.yaml b/tests/templates/kuttl/metrics/10-install-prometheus.yaml index 59d5782..b44a8ad 100644 --- a/tests/templates/kuttl/metrics/10-install-prometheus.yaml +++ b/tests/templates/kuttl/metrics/10-install-prometheus.yaml @@ -2,13 +2,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - # Append the namespace to the Helm release name so that cluster-wide - # resources get unique names. - - script: > - helm install prometheus-$NAMESPACE - --namespace $NAMESPACE - --version 80.14.2 - --values 10_kube-prometheus-stack-values.yaml - --wait - oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack + # Append the namespace to the Helm release name so that cluster-wide resources get unique names. + - script: | + PROMETHEUS_HELM_RELEASE_NAME=prometheus-$NAMESPACE + + helm install $PROMETHEUS_HELM_RELEASE_NAME \ + --namespace $NAMESPACE \ + --version 81.2.2 \ + --values 10_kube-prometheus-stack-values.yaml \ + --wait \ + oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack timeout: 600 diff --git a/tests/templates/kuttl/smoke/11-assert.yaml.j2 b/tests/templates/kuttl/smoke/11-assert.yaml.j2 index 2db20d3..65617e9 100644 --- a/tests/templates/kuttl/smoke/11-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/11-assert.yaml.j2 @@ -741,6 +741,7 @@ data: network.host: "0.0.0.0" node.attr.role-group: "cluster-manager" node.store.allow_mmap: "false" + path.logs: "/stackable/log/opensearch" plugins.security.allow_default_init_securityindex: "true" plugins.security.nodes_dn: ["CN=generated certificate for pod"] {% if test_scenario['values']['server-use-tls'] == 'true' %} @@ -781,6 +782,7 @@ data: network.host: "0.0.0.0" node.attr.role-group: "data" node.store.allow_mmap: "false" + path.logs: "/stackable/log/opensearch" plugins.security.allow_default_init_securityindex: "true" plugins.security.nodes_dn: ["CN=generated certificate for pod"] {% if test_scenario['values']['server-use-tls'] == 'true' %}