Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 361b7e9

Browse files
cccs-cat001adutra
andauthored
Added interface for reporting metrics (#2887)
Co-authored-by: Alexandre Dutra <adutra@apache.org>
1 parent 7023308 commit 361b7e9

File tree

10 files changed

+209
-2
lines changed

10 files changed

+209
-2
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
2929

3030
### Highlights
3131

32+
- Support for [Iceberg Metrics Reporting] has been introduced in Polaris. Out of the box, metrics can
33+
be printed to the logs by setting the `org.apache.polaris.service.reporting` logger level to `INFO` (it's
34+
set to `OFF` by default). Custom reporters can be implemented and configured to send metrics to
35+
external systems for further analysis and monitoring.
36+
37+
[Iceberg Metrics Reporting]: https://iceberg.apache.org/docs/latest/metrics-reporting/
38+
3239
### Upgrade notes
3340

3441
- The legacy management endpoints at `/metrics` and `/healthcheck` have been removed. Please use the

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,14 @@
6565
import org.apache.iceberg.expressions.Expressions;
6666
import org.apache.iceberg.io.FileIO;
6767
import org.apache.iceberg.io.ResolvingFileIO;
68+
import org.apache.iceberg.metrics.ImmutableScanReport;
69+
import org.apache.iceberg.metrics.ScanMetrics;
70+
import org.apache.iceberg.metrics.ScanMetricsResult;
71+
import org.apache.iceberg.metrics.ScanReport;
6872
import org.apache.iceberg.rest.RESTCatalog;
6973
import org.apache.iceberg.rest.RESTUtil;
7074
import org.apache.iceberg.rest.requests.CreateTableRequest;
75+
import org.apache.iceberg.rest.requests.ReportMetricsRequest;
7176
import org.apache.iceberg.rest.responses.ErrorResponse;
7277
import org.apache.iceberg.rest.responses.ListNamespacesResponse;
7378
import org.apache.iceberg.rest.responses.ListTablesResponse;
@@ -882,6 +887,27 @@ public void testCreateAndLoadTableWithReturnedEtag() {
882887
}
883888
}
884889

890+
@Test
891+
public void testSendMetricsReport() {
892+
ScanReport scanReport =
893+
ImmutableScanReport.builder()
894+
.tableName("tbl1")
895+
.schemaId(4)
896+
.addProjectedFieldIds(1, 2, 3)
897+
.addProjectedFieldNames("c1", "c2", "c3")
898+
.snapshotId(23L)
899+
.filter(Expressions.alwaysTrue())
900+
.scanMetrics(ScanMetricsResult.fromScanMetrics(ScanMetrics.noop()))
901+
.build();
902+
Invocation.Builder metricEndpoint =
903+
catalogApi.request(
904+
"v1/{cat}/namespaces/ns1/tables/tbl1/metrics", Map.of("cat", currentCatalogName));
905+
try (Response response =
906+
metricEndpoint.post(Entity.json(ReportMetricsRequest.of(scanReport)))) {
907+
assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus);
908+
}
909+
}
910+
885911
@Test
886912
public void testSendNotificationInternalCatalog() {
887913
Map<String, String> payload =

‎runtime/defaults/src/main/resources/application.properties‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,11 @@ polaris.oidc.principal-roles-mapper.type=default
242242
# Polaris Credential Manager Config
243243
polaris.credential-manager.type=default
244244

245+
# Configuration for the behaviour of the metrics endpoint
246+
polaris.iceberg-metrics.reporting.type=default
247+
# Set to INFO if you want to see iceberg metric reports logged
248+
quarkus.log.category."org.apache.polaris.service.reporting".level=OFF
249+
245250
quarkus.arc.ignored-split-packages=\
246251
org.apache.polaris.service.catalog.api,\
247252
org.apache.polaris.service.catalog.api.impl,\

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
import org.apache.polaris.service.events.listeners.PolarisEventListener;
8888
import org.apache.polaris.service.http.IcebergHttpUtil;
8989
import org.apache.polaris.service.http.IfNoneMatch;
90+
import org.apache.polaris.service.reporting.PolarisMetricsReporter;
9091
import org.apache.polaris.service.types.CommitTableRequest;
9192
import org.apache.polaris.service.types.CommitViewRequest;
9293
import org.apache.polaris.service.types.NotificationRequest;
@@ -150,6 +151,7 @@ public class IcebergCatalogAdapter
150151
private final Instance<ExternalCatalogFactory> externalCatalogFactories;
151152
private final PolarisEventListener polarisEventListener;
152153
private final AccessConfigProvider accessConfigProvider;
154+
private final PolarisMetricsReporter metricsReporter;
153155

154156
@Inject
155157
public IcebergCatalogAdapter(
@@ -167,7 +169,8 @@ public IcebergCatalogAdapter(
167169
CatalogHandlerUtils catalogHandlerUtils,
168170
@Any Instance<ExternalCatalogFactory> externalCatalogFactories,
169171
PolarisEventListener polarisEventListener,
170-
AccessConfigProvider accessConfigProvider) {
172+
AccessConfigProvider accessConfigProvider,
173+
PolarisMetricsReporter metricsReporter) {
171174
this.diagnostics = diagnostics;
172175
this.realmContext = realmContext;
173176
this.callContext = callContext;
@@ -184,6 +187,7 @@ public IcebergCatalogAdapter(
184187
this.externalCatalogFactories = externalCatalogFactories;
185188
this.polarisEventListener = polarisEventListener;
186189
this.accessConfigProvider = accessConfigProvider;
190+
this.metricsReporter = metricsReporter;
187191
}
188192

189193
/**
@@ -755,6 +759,11 @@ public Response reportMetrics(
755759
ReportMetricsRequest reportMetricsRequest,
756760
RealmContext realmContext,
757761
SecurityContext securityContext) {
762+
String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix);
763+
Namespace ns = decodeNamespace(namespace);
764+
TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table));
765+
766+
metricsReporter.reportMetric(catalogName, tableIdentifier, reportMetricsRequest.report());
758767
return Response.status(Response.Status.NO_CONTENT).build();
759768
}
760769

‎runtime/service/src/main/java/org/apache/polaris/service/config/ServiceProducers.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@
8383
import org.apache.polaris.service.ratelimiter.RateLimiterFilterConfiguration;
8484
import org.apache.polaris.service.ratelimiter.TokenBucketConfiguration;
8585
import org.apache.polaris.service.ratelimiter.TokenBucketFactory;
86+
import org.apache.polaris.service.reporting.MetricsReportingConfiguration;
87+
import org.apache.polaris.service.reporting.PolarisMetricsReporter;
8688
import org.apache.polaris.service.secrets.SecretsManagerConfiguration;
8789
import org.apache.polaris.service.storage.StorageConfiguration;
8890
import org.apache.polaris.service.storage.aws.S3AccessConfig;
@@ -441,4 +443,11 @@ public PolarisCredentialManager polarisCredentialManager(
441443
public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) {
442444
executor.close();
443445
}
446+
447+
@Produces
448+
@ApplicationScoped
449+
public PolarisMetricsReporter metricsReporter(
450+
MetricsReportingConfiguration config, @Any Instance<PolarisMetricsReporter> reporters) {
451+
return reporters.select(Identifier.Literal.of(config.type())).get();
452+
}
444453
}
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.service.reporting;
20+
21+
import com.google.common.annotations.VisibleForTesting;
22+
import io.smallrye.common.annotation.Identifier;
23+
import jakarta.enterprise.context.ApplicationScoped;
24+
import org.apache.commons.lang3.function.TriConsumer;
25+
import org.apache.iceberg.catalog.TableIdentifier;
26+
import org.apache.iceberg.metrics.MetricsReport;
27+
import org.slf4j.Logger;
28+
import org.slf4j.LoggerFactory;
29+
30+
@ApplicationScoped
31+
@Identifier("default")
32+
public class DefaultMetricsReporter implements PolarisMetricsReporter {
33+
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetricsReporter.class);
34+
35+
private final TriConsumer<String, TableIdentifier, MetricsReport> reportConsumer;
36+
37+
public DefaultMetricsReporter() {
38+
this(
39+
(catalogName, table, metricsReport) ->
40+
LOGGER.info("{}.{}: {}", catalogName, table, metricsReport));
41+
}
42+
43+
@VisibleForTesting
44+
DefaultMetricsReporter(TriConsumer<String, TableIdentifier, MetricsReport> reportConsumer) {
45+
this.reportConsumer = reportConsumer;
46+
}
47+
48+
@Override
49+
public void reportMetric(String catalogName, TableIdentifier table, MetricsReport metricsReport) {
50+
reportConsumer.accept(catalogName, table, metricsReport);
51+
}
52+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.service.reporting;
20+
21+
import io.smallrye.config.ConfigMapping;
22+
import io.smallrye.config.WithDefault;
23+
24+
@ConfigMapping(prefix = "polaris.iceberg-metrics.reporting")
25+
public interface MetricsReportingConfiguration {
26+
@WithDefault("default")
27+
String type();
28+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.service.reporting;
20+
21+
import org.apache.iceberg.catalog.TableIdentifier;
22+
import org.apache.iceberg.metrics.MetricsReport;
23+
24+
public interface PolarisMetricsReporter {
25+
public void reportMetric(String catalogName, TableIdentifier table, MetricsReport metricsReport);
26+
}
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.service.reporting;
20+
21+
import static org.mockito.Mockito.mock;
22+
import static org.mockito.Mockito.verify;
23+
24+
import org.apache.commons.lang3.function.TriConsumer;
25+
import org.apache.iceberg.catalog.TableIdentifier;
26+
import org.apache.iceberg.metrics.MetricsReport;
27+
import org.junit.jupiter.api.Test;
28+
29+
public class DefaultMetricsReporterTest {
30+
31+
@Test
32+
void testLogging() {
33+
TriConsumer<String, TableIdentifier, MetricsReport> mockConsumer = mock(TriConsumer.class);
34+
DefaultMetricsReporter reporter = new DefaultMetricsReporter(mockConsumer);
35+
String warehouse = "testWarehouse";
36+
TableIdentifier table = TableIdentifier.of("testNamespace", "testTable");
37+
MetricsReport metricsReport = mock(MetricsReport.class);
38+
39+
reporter.reportMetric(warehouse, table, metricsReport);
40+
41+
verify(mockConsumer).accept(warehouse, table, metricsReport);
42+
}
43+
}

‎runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
import org.apache.polaris.service.events.listeners.TestPolarisEventListener;
8383
import org.apache.polaris.service.identity.provider.DefaultServiceIdentityProvider;
8484
import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory;
85+
import org.apache.polaris.service.reporting.DefaultMetricsReporter;
8586
import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory;
8687
import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl;
8788
import org.apache.polaris.service.task.TaskExecutor;
@@ -281,7 +282,8 @@ public TestServices build() {
281282
catalogHandlerUtils,
282283
externalCatalogFactory,
283284
polarisEventListener,
284-
accessConfigProvider);
285+
accessConfigProvider,
286+
new DefaultMetricsReporter());
285287

286288
IcebergRestCatalogApi restApi = new IcebergRestCatalogApi(catalogService);
287289
IcebergRestConfigurationApi restConfigurationApi =

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /