Merge branch '285-determine-pool-name' into 'master'

feat: extract pool name while automatic detection (#285)

Closes #285

See merge request postgres-ai/database-lab!322

Author: agneum

pkg/services/provision/docker/docker_test.go

@@ -42,6 +42,7 @@ func TestVolumesBuilding(t *testing.T) {
 				CloneName: "dblab_clone_6000",
 				Pool: resources.Pool{
 					Name:         "dblab_pool",
+					PoolDirName:  "dblab_pool",
 					MountDir:     "/var/lib/dblab/",
 					CloneSubDir:  "clones",
 					DataSubDir:   "data",

pkg/services/provision/pool/pool_manager.go

@@ -19,6 +19,7 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/retrieval/engine/postgres/tools"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/resources"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/runners"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/thinclones/zfs"
 )
 
 const (
@@ -181,6 +182,8 @@ func (pm *Manager) examineEntries(entries []os.FileInfo) (map[string]FSManager,
 	fsManagers := make(map[string]FSManager)
 	poolList := &list.List{}
 
+	poolMappings := make(map[string]string)
+
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
@@ -204,6 +207,7 @@ func (pm *Manager) examineEntries(entries []os.FileInfo) (map[string]FSManager,
 		pool := &resources.Pool{
 			Mode:           fsType,
 			Name:           entry.Name(),
+			PoolDirName:    entry.Name(),
 			MountDir:       pm.cfg.MountDir,
 			CloneSubDir:    pm.cfg.CloneSubDir,
 			DataSubDir:     pm.cfg.DataSubDir,
@@ -223,6 +227,21 @@ func (pm *Manager) examineEntries(entries []os.FileInfo) (map[string]FSManager,
 			log.Msg(pool.DSA.String())
 		}
 
+		// A custom pool name is not available for LVM.
+		if fsType == ZFS {
+			if len(poolMappings) == 0 {
+				poolMappings, err = zfs.PoolMappings(pm.runner, pm.cfg.MountDir, pm.cfg.PreSnapshotSuffix)
+				if err != nil {
+					log.Msg("failed to get pool mappings:", err.Error())
+					continue
+				}
+			}
+
+			if poolName, ok := poolMappings[entry.Name()]; ok {
+				pool.Name = poolName
+			}
+		}
+
 		fsm, err := NewManager(pm.runner, ManagerConfig{
 			Pool:              pool,
 			PreSnapshotSuffix: pm.cfg.PreSnapshotSuffix,
@@ -232,8 +251,7 @@ func (pm *Manager) examineEntries(entries []os.FileInfo) (map[string]FSManager,
 			continue
 		}
 
-		// TODO(akartasov): extract pool name.
-		fsManagers[entry.Name()] = fsm
+		fsManagers[pool.Name] = fsm
 
 		front := poolList.Front()
 		if front == nil || front.Value == nil || fsManagers[front.Value.(string)].Pool().DSA.Before(pool.DSA) {

pkg/services/provision/resources/pool.go

@@ -16,6 +16,7 @@ type Pool struct {
 	Name           string
 	Mode           string
 	DSA            time.Time
+	PoolDirName    string
 	MountDir       string
 	CloneSubDir    string
 	DataSubDir     string
@@ -40,12 +41,12 @@ func (p Pool) SetDSA(dsa time.Time) {
 
 // DataDir returns a path to the data directory of the storage pool.
 func (p Pool) DataDir() string {
-	return path.Join(p.MountDir, p.Name, p.DataSubDir)
+	return path.Join(p.MountDir, p.PoolDirName, p.DataSubDir)
 }
 
 // SocketDir returns a path to the sockets directory of the storage pool.
 func (p Pool) SocketDir() string {
-	return path.Join(p.MountDir, p.Name, p.SocketSubDir)
+	return path.Join(p.MountDir, p.PoolDirName, p.SocketSubDir)
 }
 
 // ObserverDir returns a path to the observer directory of the storage pool.
@@ -55,12 +56,12 @@ func (p Pool) ObserverDir(port uint) string {
 
 // ClonesDir returns a path to the clones directory of the storage pool.
 func (p Pool) ClonesDir() string {
-	return path.Join(p.MountDir, p.Name, p.CloneSubDir)
+	return path.Join(p.MountDir, p.PoolDirName, p.CloneSubDir)
 }
 
 // ClonePath returns a path to the initialized clone directory.
 func (p Pool) ClonePath(port uint) string {
-	return path.Join(p.MountDir, p.Name, p.CloneSubDir, util.GetCloneName(port), p.DataSubDir)
+	return path.Join(p.MountDir, p.PoolDirName, p.CloneSubDir, util.GetCloneName(port), p.DataSubDir)
 }
 
 // SocketCloneDir returns a path to the socket clone directory.

pkg/services/provision/thinclones/zfs/zfs.go

@@ -7,6 +7,7 @@ package zfs
 
 import (
 	"fmt"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -655,3 +656,45 @@ func (z *ListEntry) setDataStateAt(field string) error {
 
 	return nil
 }
+
+// PoolMappings provides a mapping of pool name and mount point directory.
+func PoolMappings(runner runners.Runner, mountDir, preSnapshotSuffix string) (map[string]string, error) {
+	listCmd := "zfs list -Ho name,mountpoint -t filesystem | grep -v " + preSnapshotSuffix
+
+	output, err := runner.Run(listCmd, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list pools")
+	}
+
+	return processMappingOutput(output, mountDir), nil
+}
+
+func processMappingOutput(commandOutput, mountDir string) map[string]string {
+	poolMappings := make(map[string]string)
+
+	lines := strings.Split(commandOutput, "\n")
+
+	for i := range lines {
+		fields := strings.Fields(lines[i])
+
+		const poolFieldsNum = 2
+
+		if len(fields) < poolFieldsNum {
+			log.Dbg("Mapping fields not found: ", fields)
+			continue
+		}
+
+		// Select pools from the first nested level.
+		trimPrefix := strings.TrimPrefix(fields[1], mountDir)
+		poolDir := strings.Trim(trimPrefix, "/")
+		baseDir := path.Base(fields[1])
+
+		if poolDir == "" || baseDir != poolDir {
+			continue
+		}
+
+		poolMappings[poolDir] = fields[0]
+	}
+
+	return poolMappings
+}

pkg/services/provision/thinclones/zfs/zfs_test.go

@@ -159,3 +159,28 @@ func TestExcludingBusySnapshots(t *testing.T) {
 		assert.Equal(t, tc.result, excludeBusySnapshots(tc.snapshotList))
 	}
 }
+
+func TestProcessingMappingOutput(t *testing.T) {
+	out := `pgclusters     	           			/var/lib/postgresql/pools
+pgclusters/dblab           			/var/lib/postgresql/pools/dblab
+pgclusters/pgcluster1      			/var/lib/postgresql/pools/pgcluster1
+pgclusters/pgcluster2      			/var/lib/postgresql/pools/pgcluster2
+pgclusters/pgcluster5      			/var/lib/postgresql/pools/pgcluster5/data
+datastore                           /var/lib/postgresql/pools/datastore
+datastore/clone_pre_20210729130000  /var/lib/postgresql/pools/datastore/clones/clone_pre_20210729130000
+datastore/dblab_clone_6000          /var/lib/postgresql/pools/datastore/clones/dblab_clone_6000
+datastore/dblab_clone_6001          /var/lib/postgresql/pools/datastore/clones/dblab_clone_6001
+poolnext 							/var/lib/postgresql/pools/pool5`
+	mountDir := "/var/lib/postgresql/pools"
+	expected := map[string]string{
+		"dblab":      "pgclusters/dblab",
+		"pgcluster1": "pgclusters/pgcluster1",
+		"pgcluster2": "pgclusters/pgcluster2",
+		"datastore":  "datastore",
+		"pool5":      "poolnext",
+	}
+
+	poolMappings := processMappingOutput(out, mountDir)
+	assert.Equal(t, len(expected), len(poolMappings))
+	assert.Equal(t, expected, poolMappings)
+}