DATA_SOURCE_USER/PW/SERVICENAME support and multi-database support (#18)

paulparkinson · web-flow · commit 47097b694a33 · 2023-04-27T19:14:33.000-04:00
* add list of default metrics in readme

* readme updates

* various updates

* README.md updates

* README.md updates

* basic example

* adds... 1. ability to specify DATA_SOURCE_USER, DATA_SOURCE_PASSWORD, and DATA_SOURCE_SERVICENAME individually rather than as one DATA_SOURCE_NAME entry (thus more secure) 2. multi-database support via "/scrape" endpoint with "target" param taking dns for specific database (this will soon be enhanced so that it is instead configurable, again to be more secure) . Also, updated POM entries for 21.7 version.
diff --git a/build.sh b/build.sh
@@ -15,3 +15,4 @@ export IMAGE=${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}
 
 mvn clean package -DskipTests
 docker build -t=$IMAGE .
+docker push $IMAGE
diff --git a/examples/metrics/default-metrics.toml b/examples/metrics/default-metrics.toml
@@ -0,0 +1,6 @@
+[[metric]]
+context = "sessions"
+labels = ["inst_id", "status", "type"]
+metricsdesc = { value = "Gauge metric with count of sessions by status and type." }
+request = "select inst_id, status, type, count(*) as value from gv$session group by status, type, inst_id"
+ignorezeroresult = true
diff --git a/pom.xml b/pom.xml
@@ -15,6 +15,7 @@
 	<description>Exporter for metrics, logs, and tracing from Oracle database</description>
 	<properties>
 		<java.version>11</java.version>
+		<oracle.jdbc.version>21.7.0.0</oracle.jdbc.version>
 	</properties>
 	<dependencies>
 		<dependency>
@@ -28,9 +29,28 @@
 		</dependency>
 		<dependency>
 			<groupId>com.oracle.database.jdbc</groupId>
-			<artifactId>ojdbc11-production</artifactId>
-			<version>21.3.0.0</version>
-			<type>pom</type>
+			<artifactId>ojdbc11</artifactId>
+			<version>${oracle.jdbc.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.oracle.database.jdbc</groupId>
+			<artifactId>ucp</artifactId>
+			<version>${oracle.jdbc.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.oracle.database.security</groupId>
+			<artifactId>oraclepki</artifactId>
+			<version>${oracle.jdbc.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.oracle.database.security</groupId>
+			<artifactId>osdt_core</artifactId>
+			<version>${oracle.jdbc.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.oracle.database.security</groupId>
+			<artifactId>osdt_cert</artifactId>
+			<version>${oracle.jdbc.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>io.opentelemetry</groupId>
diff --git a/src/main/java/oracle/observability/ObservabilityExporter.java b/src/main/java/oracle/observability/ObservabilityExporter.java
@@ -6,21 +6,32 @@
 import com.oracle.bmc.secrets.model.Base64SecretBundleContentDetails;
 import com.oracle.bmc.secrets.requests.GetSecretBundleRequest;
 import com.oracle.bmc.secrets.responses.GetSecretBundleResponse;
+import oracle.observability.metrics.MetricsExporter;
 import oracle.ucp.jdbc.PoolDataSource;
 import oracle.ucp.jdbc.PoolDataSourceFactory;
 import org.apache.commons.codec.binary.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
 
 public class ObservabilityExporter {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(ObservabilityExporter.class);
     public String DEFAULT_METRICS = System.getenv("DEFAULT_METRICS"); // "default-metrics.toml"
+    public File DEFAULT_METRICS_FILE;
     public String CUSTOM_METRICS      = System.getenv("CUSTOM_METRICS"); //
     public String QUERY_TIMEOUT       = System.getenv("QUERY_TIMEOUT"); // "5"
     public String DATABASE_MAXIDLECONNS       = System.getenv("DATABASE_MAXIDLECONNS"); // "0"
     public String DATABASE_MAXOPENCONNS       = System.getenv("DATABASE_MAXOPENCONNS"); // "10"
-    public String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
+    public static String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
+    public static String DATA_SOURCE_USER = System.getenv("DATA_SOURCE_USER"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
+    public static String DATA_SOURCE_PASSWORD = System.getenv("DATA_SOURCE_PASSWORD"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
+    public static String DATA_SOURCE_SERVICENAME = System.getenv("DATA_SOURCE_SERVICENAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
     public String TNS_ADMIN = System.getenv("TNS_ADMIN");  //eg /msdataworkshop/creds
     public String OCI_REGION = System.getenv("OCI_REGION");  //eg us-ashburn-1
     public String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID");  //eg ocid....
@@ -29,27 +40,52 @@ public class ObservabilityExporter {
     public static final String CONTEXT = "context";
     public static final String REQUEST = "request";
 
+    static {
+
+        if (DATA_SOURCE_USER != null && DATA_SOURCE_PASSWORD != null && DATA_SOURCE_SERVICENAME != null) {
+            DATA_SOURCE_NAME = DATA_SOURCE_USER + "/" + DATA_SOURCE_PASSWORD + "@" + DATA_SOURCE_SERVICENAME;
+            LOGGER.info("DATA_SOURCE_NAME = DATA_SOURCE_USER + \"/\" + DATA_SOURCE_PASSWORD + \"@\" + DATA_SOURCE_SERVICENAME");
+            //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
+        }
+    }
     PoolDataSource observabilityDB;
+    Map<String, PoolDataSource> dataSourceNameToDataSourceMap = new HashMap<>();
+
     public PoolDataSource getPoolDataSource() throws SQLException {
-        if (observabilityDB != null) return observabilityDB;
-        observabilityDB = PoolDataSourceFactory.getPoolDataSource();
-        observabilityDB.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
-        String user = DATA_SOURCE_NAME.substring(0, DATA_SOURCE_NAME.indexOf("/"));
-        String pw = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("/") + 1, DATA_SOURCE_NAME.indexOf("@"));
-        String serviceName = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("@") + 1);
+        return getPoolDataSource(DATA_SOURCE_NAME);
+    }
+    public PoolDataSource getPoolDataSource(String dataSourceName) throws SQLException {
+        if (dataSourceName.equals(DATA_SOURCE_NAME)) {
+            if (observabilityDB != null) return observabilityDB;
+            return observabilityDB = getDataSource(DATA_SOURCE_NAME);
+        } else {
+            if(dataSourceNameToDataSourceMap.containsKey(dataSourceName) && dataSourceNameToDataSourceMap.get(dataSourceName) != null)
+                return dataSourceNameToDataSourceMap.get(dataSourceName);
+            PoolDataSource poolDataSource = getDataSource(dataSourceName);
+            dataSourceNameToDataSourceMap.put(dataSourceName, poolDataSource);
+            return poolDataSource;
+        }
+    }
+
+    private PoolDataSource getDataSource(String dataSourceName) throws SQLException {
+        PoolDataSource poolDataSource = PoolDataSourceFactory.getPoolDataSource();
+        poolDataSource.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
+        String user = dataSourceName.substring(0, dataSourceName.indexOf("/"));
+        String pw = dataSourceName.substring(dataSourceName.indexOf("/") + 1, dataSourceName.indexOf("@"));
+        String serviceName = dataSourceName.substring(dataSourceName.indexOf("@") + 1);
         String url = "jdbc:oracle:thin:@" + serviceName + "?TNS_ADMIN=" + TNS_ADMIN;
-        observabilityDB.setURL(url);
-        observabilityDB.setUser(user);
-        if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("")) {
-            observabilityDB.setPassword(pw);
+        poolDataSource.setURL(url);
+        poolDataSource.setUser(user);
+        if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("") || !dataSourceName.equals(DATA_SOURCE_NAME)) {
+            poolDataSource.setPassword(pw);
         } else {
             try {
-                observabilityDB.setPassword(getPasswordFromVault());
+                poolDataSource.setPassword(getPasswordFromVault());
             } catch (IOException e) {
                 throw new SQLException(e);
             }
         }
-        return observabilityDB;
+        return poolDataSource;
     }
 
 
diff --git a/src/main/java/oracle/observability/metrics/MetricsExporter.java b/src/main/java/oracle/observability/metrics/MetricsExporter.java
@@ -7,6 +7,7 @@
 import io.prometheus.client.Gauge;
 import oracle.observability.ObservabilityExporter;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import org.slf4j.Logger;
@@ -31,30 +32,38 @@ public class MetricsExporter extends ObservabilityExporter {
     public static final String LABELS = "labels";
     public static final String IGNOREZERORESULT = "ignorezeroresult";
     public static final String FALSE = "false";
-    public String LISTEN_ADDRESS      = System.getenv("LISTEN_ADDRESS"); // ":9161"
-    public String TELEMETRY_PATH         = System.getenv("TELEMETRY_PATH"); // "/metrics"
-    //Interval between each scrape. Default is to scrape on collect requests. scrape.interval
-    public String SCRAPE_INTERVAL     = System.getenv("scrape.interval"); // "0s"
     public static final String ORACLEDB_METRIC_PREFIX = "oracledb_";
     Map<String, Gauge> gaugeMap = new HashMap<>();
+    Map<String, CollectorRegistry> dnsToCollectorRegistryMap = new HashMap<>();
+
+
 
     /**
      * The endpoint that prometheus will scrape
      * @return Prometheus metric
-     * @throws Exception
      */
     @GetMapping(value = "/metrics", produces = "text/plain")
     public String metrics() throws Exception {
-        processMetrics();
-        return getMetricsString();
+        processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry);
+        return getMetricsString(CollectorRegistry.defaultRegistry);
+    }
+    @GetMapping(value = "/scrape", produces = "text/plain")
+    public String scrape(@RequestParam("target") String target) throws Exception {
+        CollectorRegistry collectorRegistry = dnsToCollectorRegistryMap.get(target);
+        if (collectorRegistry == null) {
+            collectorRegistry = new CollectorRegistry();
+            dnsToCollectorRegistryMap.put(target, collectorRegistry);
+        }
+        processMetrics(target, dnsToCollectorRegistryMap.get(target));
+        return getMetricsString(collectorRegistry);
     }
 
     @PostConstruct
     public void init() throws Exception {
-        processMetrics();
+        processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry);
     }
 
-    private void processMetrics() throws IOException, SQLException {
+    private void processMetrics(String datasourceName, CollectorRegistry registry) throws IOException, SQLException {
         File tomlfile = new File(DEFAULT_METRICS);
         TomlMapper mapper = new TomlMapper();
         JsonNode jsonNode = mapper.readerFor(MetricsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile));
@@ -65,15 +74,15 @@ private void processMetrics() throws IOException, SQLException {
         }
         Iterator<JsonNode> metrics = metric.iterator();
         int isConnectionSuccessful = 0;
-        try(Connection connection = getPoolDataSource().getConnection()) {
+        try(Connection connection = getPoolDataSource(datasourceName).getConnection()) {
             isConnectionSuccessful = 1;
             while (metrics.hasNext()) {
-                processMetric(connection, metrics);
+                processMetric(registry, connection, metrics);
             }
         } finally {
             Gauge gauge = gaugeMap.get(ORACLEDB_METRIC_PREFIX + UP);
             if (gauge == null) {
-                Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register();
+                Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register(registry);
                 upgauge.set(isConnectionSuccessful);
                 gaugeMap.put(ORACLEDB_METRIC_PREFIX + UP, upgauge);
             } else gauge.set(isConnectionSuccessful);
@@ -91,7 +100,7 @@ private void processMetrics() throws IOException, SQLException {
      * Request          string
      * IgnoreZeroResult bool
      */
-    private void processMetric(Connection connection, Iterator<JsonNode> metric) {
+    private void processMetric(CollectorRegistry registry, Connection connection, Iterator<JsonNode> metric) {
         JsonNode next = metric.next();
         String context = next.get(CONTEXT).asText(); // eg context = "teq"
         String metricsType = next.get(METRICSTYPE) == null ? "" :next.get(METRICSTYPE).asText();
@@ -120,34 +129,27 @@ private void processMetric(Connection connection, Iterator<JsonNode> metric) {
         try {
              resultSet = connection.prepareStatement(request).executeQuery();
              while (resultSet.next()) {
-                 translateQueryToPrometheusMetric(context,  metricsDescMap, labelNames, resultSet);
+                 translateQueryToPrometheusMetric(registry, context,  metricsDescMap, labelNames, resultSet);
              }
         } catch(SQLException e) { //this can be due to table not existing etc.
             LOGGER.warn("MetricsExporter.processMetric  during:" + request + " exception:" + e);
             return;
         }
     }
 
-    private void translateQueryToPrometheusMetric(String context, Map<String, String> metricsDescMap,
+    private void translateQueryToPrometheusMetric(CollectorRegistry registry, String context, Map<String, String> metricsDescMap,
                                                   String[] labelNames,
                                                   ResultSet resultSet) throws SQLException {
         String[] labelValues = new String[labelNames.length];
         Map<String, Long> sqlQueryResults =
-                extractGaugesAndLabelValues(context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount());
+                extractGaugesAndLabelValues(registry, context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount());
         setLabelValues(context, labelNames, labelValues, sqlQueryResults.entrySet().iterator());
     }
 
     /**
      * Creates Gauges and gets label values
-     * @param context
-     * @param metricsDescMap
-     * @param labelNames
-     * @param resultSet
-     * @param labelValues
-     * @param columnCount
-     * @throws SQLException
      */
-    private Map<String, Long> extractGaugesAndLabelValues(
+    private Map<String, Long> extractGaugesAndLabelValues(CollectorRegistry registry,
             String context, Map<String, String> metricsDescMap, String[] labelNames, ResultSet resultSet,
             String[] labelValues, int columnCount) throws SQLException {
         Map<String, Long> sqlQueryResults = new HashMap<>();
@@ -166,8 +168,8 @@ private Map<String, Long> extractGaugesAndLabelValues(
             if (gauge == null) {
                 if(metricsDescMap.containsKey(columnName)) {
                     if (labelNames.length > 0) {
-                        gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register();
-                    } else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register();
+                        gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register(registry);
+                    } else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register(registry);
                     gaugeMap.put(gaugeName, gauge);
                 }
             }
@@ -198,8 +200,7 @@ private void setLabelValues(String context, String[] labelNames, String[] labelV
         }
     }
 
-    public static String getMetricsString() {
-        CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
+    public static String getMetricsString(CollectorRegistry collectorRegistry) {
         Enumeration<Collector.MetricFamilySamples> mfs = collectorRegistry.filteredMetricFamilySamples(new HashSet<>());
         return compose(mfs);
     }

Original file line number	Diff line number	Diff line change
`@@ -15,3 +15,4 @@ export IMAGE=${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}`
`15`	`15`
`16`	`16`	`mvn clean package -DskipTests`
`17`	`17`	`docker build -t=$IMAGE .`
	`18`	`+docker push $IMAGE`