various (#4)

* add list of default metrics in readme

* readme updates

* various updates

Author: paulparkinson

README.md

@@ -15,6 +15,8 @@ All three exporters (metrics, log, and trace) can be configured in the same file
 
 The old version of the metrics exporter can be found in the [old implementation branch][old implementation branch] and the new metrics exporter implementation is backward compatible such that the same configuration for both database connection and metrics definition can be used.
 
+Users are encouraged to open issues and enhancements requests against this github repos and feel free to ask any questions.  We will actively work on them as we will the development of the exporters.
+
 ### Build
 
 Build without running tests using the following.

examples/README.md

@@ -4,22 +4,3 @@
 Please refer to the Unified Observability in Grafana with converged Oracle Database Workshop at http://bit.ly/unifiedobservability and it's corresponding repos https://github.com/oracle/microservices-datadriven/tree/main/grabdish/observability/db-metrics-exporter for complete examples.
 
 More examples will be provided here in the near future.
-
-# Metrics exporter 
-
-1. Pre-req. Run setup for the GrabDish workshop including observability lab steps to install and configure Grafana and Prometheus
-2. Run `./deploy.sh` in this directory
-3. `curl http://observability-exporter-example:8080/metrics` from within cluster to see Prometheus stats
-4. View same stats from within Grafana by loading AQ dashboard
-
-The same can be done above for TEW by simply replace `aq` with `teq` in the deployment and configmap yamls
-
-Troubleshooting...
-
-kubectl port-forward prometheus-stable-kube-prometheus-sta-prometheus-0 -n msdataworkshop 9090:9090
-
-# Logs exporter
-
-# Trace exporter
-
-# Combined Metrics, Logs, and Trace exporter 

examples/deploy.sh

@@ -26,6 +26,8 @@ public class ObservabilityExporter {
     public String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID");  //eg ocid....
     public String OCI_CONFIG_FILE = System.getenv("OCI_CONFIG_FILE");  //eg "~/.oci/config"
     public String OCI_PROFILE = System.getenv("OCI_PROFILE");  //eg "DEFAULT"
+    public static final String CONTEXT = "context";
+    public static final String REQUEST = "request";
 
     PoolDataSource observabilityDB;
     public PoolDataSource getPoolDataSource() throws SQLException {

examples/observability-exporter-example-service-monitor.yaml

@@ -18,12 +18,16 @@
 @RestController
 public class LogsExporter extends ObservabilityExporter implements Runnable {
 
-	private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(LogsExporter.class);
+	private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(LogsExporter.class);
+	public static final String TIMESTAMPFIELD = "timestampfield";
+	public static final String LOG = "log";
 	public String LOG_INTERVAL     = System.getenv("LOG_INTERVAL"); // "30s"
 	private int logInterval = 30;
 	List<String> lastLogged = new ArrayList<>();
 	private java.sql.Timestamp alertLogQueryLastLocalDateTime;
 
+	private int consecutiveExceptionCount = 0; //used to backoff todo should be a finer/log entry level rather than global
+
 
 	@PostConstruct
 	public void init() throws Exception {
@@ -34,26 +38,33 @@ public void init() throws Exception {
 	public void run() {
 		while (true) {
 			try {
-				LOG.debug("LogExporter default metrics from:" + DEFAULT_METRICS);
+				Thread.sleep(consecutiveExceptionCount * 1000);
+				Thread.sleep(logInterval * 1000);
+				LOGGER.debug("LogsExporter default metrics from:" + DEFAULT_METRICS);
 				if(LOG_INTERVAL!=null && !LOG_INTERVAL.trim().equals("")) logInterval = Integer.getInteger(LOG_INTERVAL);
-				LOG.debug("LogExporter logInterval:" + logInterval);
+				LOGGER.debug("LogsExporter logInterval:" + logInterval);
 				File tomlfile = new File(DEFAULT_METRICS);
 				TomlMapper mapper = new TomlMapper();
 				JsonNode jsonNode = mapper.readerFor(LogsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile));
-				JsonNode log = jsonNode.get("log");
+				JsonNode log = jsonNode.get(LOG);
 				if(log == null || log.isEmpty()) {
-					LOG.info("No logs records configured");
+					LOGGER.info("No logs records configured");
 					return;
 				}
 				Iterator<JsonNode> logs = log.iterator();
 				List<String> currentLogged = new ArrayList<>();
 				try (Connection connection = getPoolDataSource().getConnection()) {
 					while (logs.hasNext()) { //for each "log" entry in toml/config...
 						JsonNode next = logs.next();
-						String request = next.get("request").asText(); // the sql query
-						LOG.debug("DBLogsExporter. request:" + request);
-						String timestampfield = next.get("timestampfield").asText(); // eg ORIGINATING_TIMESTAMP
-						LOG.debug("DBLogsExporter. timestampfield:" + timestampfield);
+						String request = next.get(REQUEST).asText(); // the sql query
+						LOGGER.debug("LogsExporter request:" + request);
+						JsonNode timestampfieldNode = next.get(TIMESTAMPFIELD);
+						if (timestampfieldNode==null) {
+							LOGGER.warn("LogsExporter entry does not contain `timestampfield' value request:" + request);
+							continue;
+						}
+						String timestampfield = timestampfieldNode.asText(); // eg ORIGINATING_TIMESTAMP
+						LOGGER.debug("LogsExporter timestampfield:" + timestampfield);
 						PreparedStatement statement = connection.prepareStatement(
 								alertLogQueryLastLocalDateTime == null ? request : request + " WHERE " + timestampfield + " > ?");
 						if(alertLogQueryLastLocalDateTime!=null) statement.setTimestamp(1, alertLogQueryLastLocalDateTime);
@@ -82,10 +93,11 @@ public void run() {
 						}
 					}
 					lastLogged = currentLogged;
+					consecutiveExceptionCount = 0;
 				}
-				Thread.sleep(logInterval * 1000);
 			} catch (Exception e) {
-				throw new RuntimeException(e);
+				consecutiveExceptionCount++;
+				LOGGER.warn("LogsExporter.processMetric exception:" + e);
 			}
 		}
 	}

examples/observability-exporter-example-service.yaml

@@ -24,13 +24,19 @@
 @RestController
 public class MetricsExporter extends ObservabilityExporter {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(MetricsExporter.class);
+    public static final String UP = "up";
+    public static final String METRICSTYPE = "metricstype";
+    public static final String METRICSDESC = "metricsdesc";
+    public static final String LABELS = "labels";
+    public static final String IGNOREZERORESULT = "ignorezeroresult";
+    public static final String FALSE = "false";
     public String LISTEN_ADDRESS      = System.getenv("LISTEN_ADDRESS"); // ":9161"
     public String TELEMETRY_PATH         = System.getenv("TELEMETRY_PATH"); // "/metrics"
     //Interval between each scrape. Default is to scrape on collect requests. scrape.interval
     public String SCRAPE_INTERVAL     = System.getenv("scrape.interval"); // "0s"
     public static final String ORACLEDB_METRIC_PREFIX = "oracledb_";
     Map<String, Gauge> gaugeMap = new HashMap<>();
-    private static final Logger LOG = LoggerFactory.getLogger(MetricsExporter.class);
 
     /**
      * The endpoint that prometheus will scrape
@@ -54,7 +60,7 @@ private void processMetrics() throws IOException, SQLException {
         JsonNode jsonNode = mapper.readerFor(MetricsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile));
         JsonNode metric = jsonNode.get("metric");
         if(metric == null || metric.isEmpty()) {
-            LOG.info("No logs records configured");
+            LOGGER.info("No logs records configured");
             return;
         }
         Iterator<JsonNode> metrics = metric.iterator();
@@ -65,11 +71,11 @@ private void processMetrics() throws IOException, SQLException {
                 processMetric(connection, metrics);
             }
         } finally {
-            Gauge gauge = gaugeMap.get(ORACLEDB_METRIC_PREFIX + "up");
+            Gauge gauge = gaugeMap.get(ORACLEDB_METRIC_PREFIX + UP);
             if (gauge == null) {
-                Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + "up").help("Whether the Oracle database server is up.").register();
+                Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register();
                 upgauge.set(isConnectionSuccessful);
-                gaugeMap.put(ORACLEDB_METRIC_PREFIX + "up", upgauge);
+                gaugeMap.put(ORACLEDB_METRIC_PREFIX + UP, upgauge);
             } else gauge.set(isConnectionSuccessful);
         }
     }
@@ -87,38 +93,37 @@ private void processMetrics() throws IOException, SQLException {
      */
     private void processMetric(Connection connection, Iterator<JsonNode> metric) {
         JsonNode next = metric.next();
-        //todo ignore case
-        String context = next.get("context").asText(); // eg context = "teq"
-        String metricsType = next.get("metricstype") == null ? "" :next.get("metricstype").asText();
-        JsonNode metricsdescNode = next.get("metricsdesc");
+        String context = next.get(CONTEXT).asText(); // eg context = "teq"
+        String metricsType = next.get(METRICSTYPE) == null ? "" :next.get(METRICSTYPE).asText();
+        JsonNode metricsdescNode = next.get(METRICSDESC);
         // eg metricsdesc = { enqueued_msgs = "Total enqueued messages.", dequeued_msgs = "Total dequeued messages.", remained_msgs = "Total remained messages."}
         Iterator<Map.Entry<String, JsonNode>> metricsdescIterator = metricsdescNode.fields();
         Map<String, String> metricsDescMap = new HashMap<>();
         while(metricsdescIterator.hasNext()) {
             Map.Entry<String, JsonNode> metricsdesc = metricsdescIterator.next();
             metricsDescMap.put(metricsdesc.getKey(), metricsdesc.getValue().asText());
         }
-        LOG.debug("context:" + context);
+        LOGGER.debug("context:" + context);
         String[] labelNames = new String[0];
-        if (next.get("labels") != null) {
-            int size = next.get("labels").size();
-            Iterator<JsonNode> labelIterator = next.get("labels").iterator();
+        if (next.get(LABELS) != null) {
+            int size = next.get(LABELS).size();
+            Iterator<JsonNode> labelIterator = next.get(LABELS).iterator();
             labelNames = new String[size];
             for (int i = 0; i < size; i++) {
                 labelNames[i] = labelIterator.next().asText();
             }
-            LOG.debug("\n");
+            LOGGER.debug("\n");
         }
-        String request = next.get("request").asText(); // the sql query
-        String ignorezeroresult = next.get("ignorezeroresult") == null ? "false" : next.get("ignorezeroresult").asText(); //todo, currently defaults to true
+        String request = next.get(REQUEST).asText(); // the sql query
+        String ignorezeroresult = next.get(IGNOREZERORESULT) == null ? FALSE : next.get(IGNOREZERORESULT).asText(); //todo, currently defaults to true
         ResultSet resultSet;
         try {
              resultSet = connection.prepareStatement(request).executeQuery();
              while (resultSet.next()) {
                  translateQueryToPrometheusMetric(context,  metricsDescMap, labelNames, resultSet);
              }
         } catch(SQLException e) { //this can be due to table not existing etc.
-            LOG.warn("DBMetricsExporter.processMetric  during:" + request + " exception:" + e);
+            LOGGER.warn("MetricsExporter.processMetric  during:" + request + " exception:" + e);
             return;
         }
     }
@@ -127,7 +132,7 @@ private void translateQueryToPrometheusMetric(String context, Map<String, String
                                                   String[] labelNames,
                                                   ResultSet resultSet) throws SQLException {
         String[] labelValues = new String[labelNames.length];
-        Map<String, Integer> sqlQueryResults =
+        Map<String, Long> sqlQueryResults =
                 extractGaugesAndLabelValues(context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount());
         setLabelValues(context, labelNames, labelValues, sqlQueryResults.entrySet().iterator());
     }
@@ -142,10 +147,10 @@ private void translateQueryToPrometheusMetric(String context, Map<String, String
      * @param columnCount
      * @throws SQLException
      */
-    private Map<String, Integer> extractGaugesAndLabelValues(
+    private Map<String, Long> extractGaugesAndLabelValues(
             String context, Map<String, String> metricsDescMap, String[] labelNames, ResultSet resultSet,
             String[] labelValues, int columnCount) throws SQLException {
-        Map<String, Integer> sqlQueryResults = new HashMap<>();
+        Map<String, Long> sqlQueryResults = new HashMap<>();
         String columnName;
         String columnTypeName;
         for (int i = 0; i < columnCount; i++) { //for each column...
@@ -154,9 +159,9 @@ private Map<String, Integer> extractGaugesAndLabelValues(
             if (columnTypeName.equals("VARCHAR2"))  //.  typename is 2/NUMBER or 12/VARCHAR2
                 ;
             else
-                sqlQueryResults.put(resultSet.getMetaData().getColumnName(i + 1), resultSet.getInt(i + 1));
+                sqlQueryResults.put(resultSet.getMetaData().getColumnName(i + 1), resultSet.getLong(i + 1));
             String gaugeName = ORACLEDB_METRIC_PREFIX + context + "_" + columnName;
-            LOG.debug("---gaugeName:" + gaugeName);
+            LOGGER.debug("---gaugeName:" + gaugeName);
             Gauge gauge = gaugeMap.get(gaugeName);
             if (gauge == null) {
                 if(metricsDescMap.containsKey(columnName)) {
@@ -173,9 +178,9 @@ private Map<String, Integer> extractGaugesAndLabelValues(
         return sqlQueryResults;
     }
 
-    private void setLabelValues(String context, String[] labelNames, String[] labelValues, Iterator<Map.Entry<String, Integer>> sqlQueryRestulsEntryIterator) {
+    private void setLabelValues(String context, String[] labelNames, String[] labelValues, Iterator<Map.Entry<String, Long>> sqlQueryRestulsEntryIterator) {
         while(sqlQueryRestulsEntryIterator.hasNext()) { //for each column
-            Map.Entry<String, Integer> sqlQueryResultsEntry =   sqlQueryRestulsEntryIterator.next();
+            Map.Entry<String, Long> sqlQueryResultsEntry =   sqlQueryRestulsEntryIterator.next();
             boolean isLabel = false;
             for (int ii = 0; ii< labelNames.length; ii++) {
                 if(labelNames[ii].equals(sqlQueryResultsEntry.getKey())) isLabel =true;  // continue
@@ -186,7 +191,7 @@ private void setLabelValues(String context, String[] labelNames, String[] labelV
                     try {
                         gaugeMap.get(ORACLEDB_METRIC_PREFIX + context + "_" + sqlQueryResultsEntry.getKey().toLowerCase()).labels(labelValues).set(valueToSet);
                     } catch (Exception ex) { //todo filter to avoid unnecessary exception handling
-                        LOG.debug("OracleDBMetricsExporter.translateQueryToPrometheusMetric Exc:" + ex);
+                        LOGGER.debug("OracleDBMetricsExporter.translateQueryToPrometheusMetric Exc:" + ex);
                     }
                 else gaugeMap.get(ORACLEDB_METRIC_PREFIX + context + "_" + sqlQueryResultsEntry.getKey().toLowerCase()).set(valueToSet);
             }