From 56792cc7302e25f98d8d365375fb7fedd260ff53 Mon Sep 17 00:00:00 2001
From: bpeng <b.peng@gns.cri.nz>
Date: Wed, 13 Aug 2025 14:42:33 +1200
Subject: [PATCH] feat: add robots.txt handler

---
 cmd/fdsn-ws/assets/robots.txt |  3 +++
 cmd/fdsn-ws/routes.go         | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 cmd/fdsn-ws/assets/robots.txt

diff --git a/cmd/fdsn-ws/assets/robots.txt b/cmd/fdsn-ws/assets/robots.txt
new file mode 100644
index 00000000..6ffbc308
--- /dev/null
+++ b/cmd/fdsn-ws/assets/robots.txt
@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: /
+
diff --git a/cmd/fdsn-ws/routes.go b/cmd/fdsn-ws/routes.go
index 00a1631d..3c635622 100644
--- a/cmd/fdsn-ws/routes.go
+++ b/cmd/fdsn-ws/routes.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	_ "embed"
 	"fmt"
 	"net/http"
 	"strings"
@@ -42,6 +43,9 @@ func initRoutes() {
 	mux.HandleFunc("/metrics/fdsnws/dataselect/1/query", weft.MakeHandler(fdsnDataMetricsV1Handler, weft.TextError))
 
 	mux.HandleFunc("/sc3ml", weft.MakeHandler(s3ml, weft.TextError))
+
+	// handle robots
+	mux.HandleFunc("/robots.txt", weft.MakeHandler(robots, weft.TextError))
 }
 
 func soh(r *http.Request, h http.Header, b *bytes.Buffer) error {
@@ -114,3 +118,18 @@ func fdsnErrorHandler(err error, h http.Header, b *bytes.Buffer, nounce string)
 
 	return weft.TextError(err, h, b, nounce)
 }
+
+//go:embed assets/robots.txt
+var robot string
+
+// robots handler for crawlers
+func robots(r *http.Request, h http.Header, b *bytes.Buffer) error {
+	h.Set("Content-Type", "text/plain")
+	h.Set("Surrogate-Control", "max-age=3600")
+
+	_, err := b.WriteString(robot)
+	if err != nil {
+		return weft.StatusError{Code: http.StatusInternalServerError, Err: err}
+	}
+	return nil
+}