fix: wss related tests are unstable (#2675) (b70e22e) (#371)

Author: github-actions[bot]

internal/controller/gateway_controller.go

@@ -120,6 +120,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		}
 		return ctrl.Result{}, err
 	}
+	if !r.checkGatewayClass(gateway) {
+		return ctrl.Result{}, nil
+	}
+
 	conditionProgrammedStatus, conditionProgrammedMsg := true, "Programmed"
 
 	r.Log.Info("gateway has been accepted", "gateway", gateway.GetName())

internal/controller/indexer/ssl_host.go

@@ -69,9 +69,11 @@ func GatewayTLSHostIndexFunc(rawObj client.Object) []string {
 		}
 	}
 
-	tlsHostIndexLogger.Info("GatewayTLSHostIndexFunc", "hosts", hostSetToSlice(hosts), "len", len(hostSetToSlice(hosts)))
+	hostsSlice := hostSetToSlice(hosts)
 
-	return hostSetToSlice(hosts)
+	tlsHostIndexLogger.V(1).Info("GatewayTLSHostIndexFunc", "hosts", hostsSlice)
+
+	return hostsSlice
 }
 
 // IngressTLSHostIndexFunc indexes Ingresses by their TLS SNI hosts.

test/e2e/crds/v2/route.go

@@ -2323,28 +2323,12 @@ spec:
 			applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"},
 				new(apiv2.ApisixRoute), fmt.Sprintf(apisixRouteWithBackendWSS, s.Namespace()))
 
-			By("verify wss connection with retry")
-			u := url.URL{
-				Scheme: "wss",
-				Host:   s.GetAPISIXHTTPSEndpoint(),
-				Path:   "/ws",
-			}
-			headers := http.Header{"Host": []string{"api6.com"}}
-			dialer := websocket.Dialer{
-				TLSClientConfig: &tls.Config{
-					InsecureSkipVerify: true,
-					ServerName:         "api6.com",
-				},
-			}
-
-			var conn *websocket.Conn
-			var resp *http.Response
-			Eventually(func() error {
-				var dialErr error
-				conn, resp, dialErr = dialer.Dial(u.String(), headers)
-				return dialErr
-			}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
-			Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
+			By("verify wss connection")
+			hostname := "api6.com"
+			conn := s.NewWebsocketClient(&tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         hostname,
+			}, "/ws", http.Header{"Host": []string{hostname}})
 
 			defer func() {
 				_ = conn.Close()

test/e2e/framework/manifests/apisix.yaml

@@ -63,15 +63,18 @@ metadata:
   name: apisix
   labels:
     app.kubernetes.io/name: apisix
+    app: apisix
 spec:
   replicas: {{ default 1 .Replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: apisix
+      app: apisix
   template:
     metadata:
       labels:
         app.kubernetes.io/name: apisix
+        app: apisix
     spec:
       initContainers:
         - name: config-setup

test/e2e/framework/manifests/nginx.yaml

@@ -124,13 +124,19 @@ spec:
       containers:
         - livenessProbe:
             failureThreshold: 3
-            initialDelaySeconds: 1
-            periodSeconds: 5
+            initialDelaySeconds: 10
+            periodSeconds: 15
             successThreshold: 1
             httpGet:
               path: /healthz
               port: 80
             timeoutSeconds: 2
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 80
+            initialDelaySeconds: 5
+            periodSeconds: 5
           image: "openresty/openresty:1.27.1.2-4-bullseye-fat"
           imagePullPolicy: IfNotPresent
           name: nginx

test/e2e/gatewayapi/httproute.go

@@ -22,7 +22,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -2507,33 +2506,13 @@ spec:
 
 		It("WSS backend", func() {
 			s.ResourceApplied("HTTPRoute", "nginx-wss", fmt.Sprintf(httprouteWithWSS, s.Namespace()), 1)
-			time.Sleep(6 * time.Second)
 
 			By("verify wss connection")
-			u := url.URL{
-				Scheme: "wss",
-				Host:   s.GetAPISIXHTTPSEndpoint(),
-				Path:   "/ws",
-			}
-			headers := http.Header{"Host": []string{"api6.com"}}
-
 			hostname := "api6.com"
-
-			dialer := websocket.Dialer{
-				TLSClientConfig: &tls.Config{
-					InsecureSkipVerify: true,
-					ServerName:         hostname,
-				},
-			}
-
-			var conn *websocket.Conn
-			var resp *http.Response
-			Eventually(func() error {
-				var dialErr error
-				conn, resp, dialErr = dialer.Dial(u.String(), headers)
-				return dialErr
-			}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
-			Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
+			conn := s.NewWebsocketClient(&tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         hostname,
+			}, "/ws", http.Header{"Host": []string{hostname}})
 
 			defer func() {
 				_ = conn.Close()

test/e2e/ingress/ingress.go

@@ -23,7 +23,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -992,14 +991,13 @@ spec:
 			By("create Ingress")
 			err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingress, s.Namespace()), s.Namespace())
 			Expect(err).NotTo(HaveOccurred(), "creating Ingress")
-			time.Sleep(5 * time.Second)
 
 			By("verify Ingress works")
-			s.NewAPISIXClient().
-				GET("/get").
-				WithHost("ingress.example.com").
-				Expect().
-				Status(200)
+			s.RequestAssert(&scaffold.RequestAssert{
+				Method: "GET",
+				Host:   "ingress.example.com",
+				Check:  scaffold.WithExpectedStatus(http.StatusOK),
+			})
 
 			By("create additional gateway group to get new admin key")
 			additionalGatewayGroupID, _, err = s.Deployer.CreateAdditionalGateway("gateway-proxy-update")
@@ -1121,30 +1119,13 @@ spec:
 			createSecret(s, _secretName)
 			By("create Ingress")
 			Expect(s.CreateResourceFromString(fmt.Sprintf(ingressWithWSS, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress")
-			time.Sleep(6 * time.Second)
 
 			By("verify wss connection")
-			u := url.URL{
-				Scheme: "wss",
-				Host:   s.GetAPISIXHTTPSEndpoint(),
-				Path:   "/ws",
-			}
-			headers := http.Header{"Host": []string{"api6.com"}}
-			dialer := websocket.Dialer{
-				TLSClientConfig: &tls.Config{
-					InsecureSkipVerify: true,
-					ServerName:         "api6.com",
-				},
-			}
-
-			var conn *websocket.Conn
-			var resp *http.Response
-			Eventually(func() error {
-				var dialErr error
-				conn, resp, dialErr = dialer.Dial(u.String(), headers)
-				return dialErr
-			}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
-			Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
+			hostname := "api6.com"
+			conn := s.NewWebsocketClient(&tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         hostname,
+			}, "/ws", http.Header{"Host": []string{hostname}})
 
 			defer func() {
 				_ = conn.Close()

test/e2e/scaffold/apisix_deployer.go

@@ -111,6 +111,19 @@ func (s *APISIXDeployer) AfterEach() {
 		if output != "" {
 			_, _ = fmt.Fprintln(GinkgoWriter, output)
 		}
+
+		_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX logs:")
+		output = s.GetDeploymentLogs("apisix")
+		if output != "" {
+			_, _ = fmt.Fprintln(GinkgoWriter, output)
+		}
+		if framework.ProviderType == framework.ProviderTypeAPISIXStandalone && s.adminTunnel != nil {
+			client := NewClient("http", s.adminTunnel.Endpoint())
+			reporter := &ErrorReporter{}
+			body := client.GET("/apisix/admin/configs").WithHeader("X-API-KEY", s.AdminKey()).WithReporter(reporter).Expect().Body().Raw()
+			_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX configs:")
+			_, _ = fmt.Fprintln(GinkgoWriter, body)
+		}
 	}
 
 	// Delete all additional gateways

test/e2e/scaffold/scaffold.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/api7/gopkg/pkg/log"
 	"github.com/gavv/httpexpect/v2"
+	"github.com/gorilla/websocket"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/testing"
 	. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
@@ -579,3 +580,37 @@ func (s *Scaffold) GetMetricsEndpoint() string {
 	s.addFinalizers(tunnel.Close)
 	return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint())
 }
+
+func (s *Scaffold) NewWebsocketClient(tls *tls.Config, path string, headers http.Header) *websocket.Conn {
+	var host = s.ApisixHTTPEndpoint()
+	var scheme = "ws"
+	if tls != nil {
+		scheme = "wss"
+		host = s.GetAPISIXHTTPSEndpoint()
+	}
+
+	dialer := websocket.Dialer{
+		TLSClientConfig: tls,
+	}
+
+	u := url.URL{
+		Scheme: scheme,
+		Host:   host,
+		Path:   path,
+	}
+	var conn *websocket.Conn
+
+	s.RetryAssertion(func() error {
+		c, resp, err := dialer.Dial(u.String(), headers)
+		if err != nil {
+			return err
+		}
+		if resp == nil || resp.StatusCode != http.StatusSwitchingProtocols {
+			_ = c.Close()
+			return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		}
+		conn = c
+		return nil
+	}).ShouldNot(HaveOccurred(), "establishing websocket connection")
+	return conn
+}