RHINENG-22325: inventory views

base/database/testing.go

@@ -400,8 +400,12 @@ func DeleteSystem(t *testing.T, inventoryID string) {
 
 func CreateTemplate(t *testing.T, account int, uuid string, inventoryIDs []string) {
 	template := &models.Template{
-		RhAccountID: account, UUID: uuid, Name: uuid, EnvironmentID: strings.ReplaceAll(uuid, "-", ""),
-		Arch: "x86_64", Version: "8",
+		TemplateBase: models.TemplateBase{
+			RhAccountID: account, UUID: uuid, Name: uuid,
+		},
+		EnvironmentID: strings.ReplaceAll(uuid, "-", ""),
+		Arch:          "x86_64",
+		Version:       "8",
 	}
 
 	tx := DB.Begin()

base/inventory_views/inventory_views.go

@@ -0,0 +1,112 @@
+package inventory_views
+
+import (
+	"app/base/models"
+	"app/base/utils"
+	"time"
+
+	"gorm.io/gorm"
+)
+
+type InventoryViewsHost struct {
+	// Inventory ID (UUID) of the host
+	ID   string                 `json:"id"`
+	Data InventoryViewsHostData `json:"data"`
+}
+
+type InventoryViewsHostData struct {
+	ApplicableRhsaCount   int     `json:"applicable_rhsa_count"`
+	ApplicableRhbaCount   int     `json:"applicable_rhba_count"`
+	ApplicableRheaCount   int     `json:"applicable_rhea_count"`
+	ApplicableOtherCount  int     `json:"applicable_other_count"`
+	InstallableRhsaCount  int     `json:"installable_rhsa_count"`
+	InstallableRhbaCount  int     `json:"installable_rhba_count"`
+	InstallableRheaCount  int     `json:"installable_rhea_count"`
+	InstallableOtherCount int     `json:"installable_other_count"`
+	PackagesInstalled     int     `json:"packages_installed"`
+	PackagesInstallable   int     `json:"packages_installable"`
+	PackagesApplicable    int     `json:"packages_applicable"`
+	TemplateName          *string `json:"template_name"`
+	TemplateUUID          *string `json:"template_uuid"`
+}
+
+type InventoryViewsEvent struct {
+	OrgID     string               `json:"org_id"`
+	Timestamp string               `json:"timestamp"`
+	Hosts     []InventoryViewsHost `json:"hosts"`
+}
+
+func MakeInventoryViewsEvent(tx *gorm.DB, orgID string, systems []models.SystemPlatform) (
+	InventoryViewsEvent, error) {
+	templates, err := FindSystemsTemplates(tx, systems)
+	if err != nil {
+		return InventoryViewsEvent{}, err
+	}
+	hosts := MakeInventoryViewsHosts(systems, templates)
+	return InventoryViewsEvent{OrgID: orgID, Timestamp: time.Now().Format(time.RFC3339), Hosts: hosts}, nil
+}
+
+func MakeInventoryViewsHosts(systems []models.SystemPlatform,
+	templates map[int64]models.TemplateBase) []InventoryViewsHost {
+	hosts := make([]InventoryViewsHost, len(systems))
+	for i, system := range systems {
+		hosts[i] = InventoryViewsHost{
+			ID: system.InventoryID,
+			Data: InventoryViewsHostData{
+				ApplicableRhsaCount: system.ApplicableAdvisorySecCountCache,
+				ApplicableRhbaCount: system.ApplicableAdvisoryBugCountCache,
+				ApplicableRheaCount: system.ApplicableAdvisoryEnhCountCache,
+				ApplicableOtherCount: system.ApplicableAdvisoryCountCache - system.ApplicableAdvisorySecCountCache -
+					system.ApplicableAdvisoryBugCountCache - system.ApplicableAdvisoryEnhCountCache,
+				InstallableRhsaCount: system.InstallableAdvisorySecCountCache,
+				InstallableRhbaCount: system.InstallableAdvisoryBugCountCache,
+				InstallableRheaCount: system.InstallableAdvisoryEnhCountCache,
+				InstallableOtherCount: system.InstallableAdvisoryCountCache - system.InstallableAdvisorySecCountCache -
+					system.InstallableAdvisoryBugCountCache - system.InstallableAdvisoryEnhCountCache,
+				PackagesInstalled:   system.PackagesInstalled,
+				PackagesInstallable: system.PackagesInstallable,
+				PackagesApplicable:  system.PackagesApplicable,
+			},
+		}
+		if system.TemplateID != nil {
+			template, ok := templates[*system.TemplateID]
+			if ok {
+				hosts[i].Data.TemplateName = &template.Name
+				hosts[i].Data.TemplateUUID = &template.UUID
+			} else {
+				utils.LogWarn("template_id", system.TemplateID, "template not found")
+			}
+		}
+	}
+	return hosts
+}
+
+func FindSystemsTemplates(tx *gorm.DB, systems []models.SystemPlatform) (map[int64]models.TemplateBase, error) {
+	templateIDs := make([]int64, 0, len(systems))
+	if len(systems) == 0 {
+		return nil, nil
+	}
+	for _, system := range systems {
+		if system.TemplateID == nil {
+			continue
+		}
+		templateIDs = append(templateIDs, *system.TemplateID)
+	}
+
+	if len(templateIDs) == 0 {
+		return nil, nil
+	}
+	templates := make([]models.TemplateBase, 0, len(templateIDs))
+	q := tx.Model(&models.TemplateBase{}).
+		Where("rh_account_id = ? AND id IN (?)", systems[0].RhAccountID, templateIDs)
+	err := q.Find(&templates).Error
+	if err != nil {
+		return nil, err
+	}
+
+	templatesMap := make(map[int64]models.TemplateBase, len(templates))
+	for _, t := range templates {
+		templatesMap[t.ID] = t
+	}
+	return templatesMap, nil
+}

base/inventory_views/inventory_views_test.go

@@ -0,0 +1,95 @@
+package inventory_views
+
+import (
+	"app/base/core"
+	"app/base/database"
+	"app/base/models"
+	"app/base/utils"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const rhAccountID = 1
+
+func TestMakeInventoryViewsEvent(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+	tx := database.DB.Begin()
+	defer tx.Rollback()
+
+	var rhAccount models.RhAccount
+	assert.NoError(t, tx.Where("id = ?", rhAccountID).First(&rhAccount).Error)
+	assert.NotEmpty(t, *rhAccount.OrgID)
+
+	var systems []models.SystemPlatform
+	assert.NoError(t, tx.Where("rh_account_id = ? AND id in (1,3)", rhAccountID).
+		Order("id").Find(&systems).Error)
+	assert.Equal(t, 2, len(systems))
+
+	event, err := MakeInventoryViewsEvent(tx, *rhAccount.OrgID, systems)
+	assert.NoError(t, err)
+	assert.Equal(t, *rhAccount.OrgID, event.OrgID)
+	assert.NotEmpty(t, event.Timestamp)
+	_, err = time.Parse(time.RFC3339, event.Timestamp)
+	assert.NoError(t, err)
+
+	// Verify hosts
+	assert.Equal(t, 2, len(event.Hosts))
+
+	assert.Equal(t, InventoryViewsHost{
+		ID: "00000000-0000-0000-0000-000000000001",
+		Data: InventoryViewsHostData{2, 3, 3, 0, 2, 2, 1, 0, 0, 0, 0,
+			utils.PtrString("temp1-1"), utils.PtrString("99900000-0000-0000-0000-000000000001")},
+	}, event.Hosts[0])
+
+	assert.Equal(t, InventoryViewsHost{
+		ID: "00000000-0000-0000-0000-000000000003",
+		Data: InventoryViewsHostData{0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0,
+			utils.PtrString("temp2-1"), utils.PtrString("99900000-0000-0000-0000-000000000002")},
+	}, event.Hosts[1])
+}
+
+func TestMakeInventoryViewsEventEmpty(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+
+	tx := database.DB.Begin()
+	defer tx.Rollback()
+
+	event, err := MakeInventoryViewsEvent(tx, "test-org", []models.SystemPlatform{})
+	assert.NoError(t, err)
+	assert.Equal(t, "test-org", event.OrgID)
+	assert.Equal(t, 0, len(event.Hosts))
+	assert.NotEmpty(t, event.Timestamp)
+}
+
+func TestMakeInventoryViewsEventNoTemplate(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+
+	tx := database.DB.Begin()
+	defer tx.Rollback()
+
+	var rhAccount models.RhAccount
+	assert.NoError(t, tx.Where("id = ?", rhAccountID).First(&rhAccount).Error)
+	assert.NotEmpty(t, *rhAccount.OrgID)
+	orgID := *rhAccount.OrgID
+
+	var systems []models.SystemPlatform
+	assert.NoError(t, tx.Where("rh_account_id = ? AND id in (4)", rhAccountID).
+		Order("id").Find(&systems).Error)
+	assert.Equal(t, 1, len(systems))
+
+	// Should not error, but template fields should be nil
+	event, err := MakeInventoryViewsEvent(tx, orgID, systems)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(event.Hosts))
+
+	host := event.Hosts[0]
+	assert.Equal(t, "00000000-0000-0000-0000-000000000004", host.ID)
+	// Template fields should be nil when template is not found
+	assert.Nil(t, host.Data.TemplateName)
+	assert.Nil(t, host.Data.TemplateUUID)
+}

base/models/models.go

@@ -25,12 +25,16 @@ func (Reporter) TableName() string {
 	return "reporter"
 }
 
+type TemplateBase struct {
+	ID          int64 `gorm:"primaryKey"`
+	RhAccountID int   `gorm:"primaryKey"`
+	UUID        string
+	Name        string
+}
+
 type Template struct {
-	ID            int64 `gorm:"primaryKey"`
-	RhAccountID   int   `gorm:"primaryKey"`
-	UUID          string
+	TemplateBase
 	EnvironmentID string
-	Name          string
 	Arch          string
 	Version       string
 	// Config      pgtype.JSONB // currently unused
@@ -40,7 +44,7 @@ type Template struct {
 	LastEdited  *time.Time
 }
 
-func (Template) TableName() string {
+func (TemplateBase) TableName() string {
 	return "template"
 }
 

base/mqueue/message.go

@@ -1,12 +1,16 @@
 package mqueue
 
-import "github.com/bytedance/sonic"
+import (
+	"github.com/bytedance/sonic"
+	"github.com/segmentio/kafka-go"
+)
 
-func MessageFromJSON(k string, v interface{}) (KafkaMessage, error) {
+func MessageFromJSON(k string, v interface{}, h []kafka.Header) (KafkaMessage, error) {
 	var m KafkaMessage
 	var err error
 
 	m.Key = []byte(k)
+	m.Headers = h
 	m.Value, err = sonic.Marshal(v)
 	return m, err
 }

base/mqueue/mqueue.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/lestrrat-go/backoff/v2"
+	"github.com/segmentio/kafka-go"
 )
 
 const errContextCanceled = "context canceled"
@@ -42,8 +43,9 @@ func createLoggerFunc(counter Counter) func(fmt string, args ...interface{}) {
 }
 
 type KafkaMessage struct {
-	Key   []byte
-	Value []byte
+	Key     []byte
+	Value   []byte
+	Headers []kafka.Header
 }
 
 type MessageHandler func(message KafkaMessage) error

base/mqueue/mqueue_impl_gokafka.go

@@ -39,7 +39,7 @@ func (t *kafkaGoReaderImpl) HandleMessages(handler MessageHandler) {
 			}
 		}
 		// At this level, all errors are fatal
-		kafkaMessage := KafkaMessage{Key: m.Key, Value: m.Value}
+		kafkaMessage := KafkaMessage{Key: m.Key, Value: m.Value, Headers: m.Headers}
 		if err = handler(kafkaMessage); err != nil {
 			utils.LogPanic("err", err.Error(), "Handler failed")
 		}
@@ -61,7 +61,7 @@ type kafkaGoWriterImpl struct {
 func (t *kafkaGoWriterImpl) WriteMessages(ctx context.Context, msgs ...KafkaMessage) error {
 	kafkaGoMessages := make([]kafka.Message, len(msgs))
 	for i, m := range msgs {
-		kafkaGoMessages[i] = kafka.Message{Key: m.Key, Value: m.Value}
+		kafkaGoMessages[i] = kafka.Message{Key: m.Key, Value: m.Value, Headers: m.Headers}
 	}
 	err := t.Writer.WriteMessages(ctx, kafkaGoMessages...)
 	return err

base/utils/config.go

@@ -66,6 +66,7 @@ type coreConfig struct {
 	RemediationUpdateTopic string
 	NotificationsTopic     string
 	TemplateTopic          string
+	InventoryViewsTopic    string
 
 	// services
 	VmaasAddress                  string
@@ -149,6 +150,11 @@ func initDBFromEnv() {
 }
 
 func initKafkaFromEnv() {
+	overrideKafkaAddress := Getenv("KAFKA_ADDRESS", "")
+	if overrideKafkaAddress != "" {
+		CoreCfg.KafkaAddress = overrideKafkaAddress
+		CoreCfg.KafkaServers = []string{overrideKafkaAddress}
+	}
 	CoreCfg.KafkaSslCert = Getenv("KAFKA_SSL_CERT", CoreCfg.KafkaSslCert)
 	CoreCfg.KafkaSslSkipVerify = GetBoolEnvOrDefault("KAFKA_SSL_SKIP_VERIFY", false)
 	CoreCfg.KafkaUsername = Getenv("KAFKA_USERNAME", CoreCfg.KafkaUsername)
@@ -167,6 +173,7 @@ func initTopicsFromEnv() {
 	CoreCfg.RemediationUpdateTopic = Getenv("REMEDIATIONS_UPDATE_TOPIC", "")
 	CoreCfg.NotificationsTopic = Getenv("NOTIFICATIONS_TOPIC", "")
 	CoreCfg.TemplateTopic = Getenv("TEMPLATE_TOPIC", "")
+	CoreCfg.InventoryViewsTopic = Getenv("INVENTORY_VIEWS_TOPIC", "")
 }
 
 func initServicesFromEnv() {
@@ -248,6 +255,7 @@ func initKafkaFromClowder() {
 		translateTopic(&CoreCfg.RemediationUpdateTopic)
 		translateTopic(&CoreCfg.NotificationsTopic)
 		translateTopic(&CoreCfg.TemplateTopic)
+		translateTopic(&CoreCfg.InventoryViewsTopic)
 	}
 }
 
@@ -399,6 +407,7 @@ func printKafkaParams() {
 	fmt.Printf("REMEDIATIONS_UPDATE_TOPIC=%s\n", CoreCfg.RemediationUpdateTopic)
 	fmt.Printf("NOTIFICATIONS_TOPIC=%s\n", CoreCfg.NotificationsTopic)
 	fmt.Printf("TEMPLATE_TOPIC=%s\n", CoreCfg.TemplateTopic)
+	fmt.Printf("INVENTORY_VIEWS_TOPIC=%s\n", CoreCfg.InventoryViewsTopic)
 }
 
 func printServicesParams() {

conf/common.env

@@ -17,6 +17,7 @@ NOTIFICATIONS_TOPIC=platform.notifications.ingress
 PAYLOAD_TRACKER_TOPIC=platform.payload-status
 REMEDIATIONS_UPDATE_TOPIC=platform.remediation-updates.patch
 TEMPLATE_TOPIC=platform.content-sources.template
+INVENTORY_VIEWS_TOPIC=platform.inventory.host-apps
 
 # If vmaas is running locally, its available here
 #VMAAS_ADDRESS=http://vmaas_webapp:8080

conf/local.env

@@ -16,13 +16,14 @@ DB_SSLROOTCERT=dev/database/secrets/pgca.crt
 VMAAS_ADDRESS=http://localhost:9001
 CANDLEPIN_ADDRESS=http://localhost:9001/candlepin
 
-#KAFKA_ADDRESS=localhost:29092
+KAFKA_ADDRESS=localhost:9092
 KAFKA_GROUP=patchman
 KAFKA_SSL_CERT=dev/kafka/secrets/ca.crt
 PAYLOAD_TRACKER_TOPIC=platform.payload-status
 EVENTS_TOPIC=platform.inventory.events
 EVAL_TOPIC=patchman.evaluator.user-evaluation
 TEMPLATE_TOPIC=platform.content-sources.template
+INVENTORY_VIEWS_TOPIC=platform.inventory.host-apps
 
 RBAC_ADDRESS=http://localhost:9001
 

conf/test.env

@@ -10,7 +10,7 @@ DB_PASSWD=passwd
 LIMIT_PAGE_SIZE=false
 
 # don't put "" or '' around the text otherwise they'll be included into content
-POD_CONFIG=label=upload;vmaas_call_max_retries=100;template_change_eval=false;update_users;update_db_config;use_testing_db
+POD_CONFIG=label=upload;vmaas_call_max_retries=100;template_change_eval=false;update_users;update_db_config;use_testing_db;inventory_views
 
 KESSEL_URL=platform:9005
 KESSEL_INSECURE=true