MM-61147: CentOS/RHEL experimental support on feature branch

api/agent.go

@@ -494,8 +494,9 @@ func getUserCredentials(usersFilePath string, _ *loadtest.Config) ([]user, error
 		password := split[1]
 		// Quick and dirty hack to extract username from email.
 		// This is not terribly important to be correct.
-		username := strings.Split(email, "@")[0]
-		username = strings.Replace(username, "+", "-", -1)
+		// username := strings.Split(email, "@")[0]
+		// username = strings.Replace(username, "+", "-", -1)
+		username := email
 		authService := userentity.AuthenticationTypeMattermost
 
 		// Check if the user has a custom authentication type. Custom authentication types are

cmd/ltctl/collect.go

@@ -113,7 +113,7 @@ func saveCollection(namePrefix string, files []file) error {
 	return nil
 }
 
-func createClients(output *terraform.Output) (map[string]*ssh.Client, error) {
+func createClients(config deployment.Config, output *terraform.Output) (map[string]*ssh.Client, error) {
 	extAgent, err := ssh.NewAgent()
 	if err != nil {
 		return nil, err
@@ -122,26 +122,26 @@ func createClients(output *terraform.Output) (map[string]*ssh.Client, error) {
 	clients := make(map[string]*ssh.Client)
 	if output.HasProxy() {
 		for i, inst := range output.Proxies {
-			sshc, err := extAgent.NewClient(inst.PrivateIP)
+			sshc, err := extAgent.NewClient(inst.PrivateIP, config.AWSAMIUser)
 			if err != nil {
-				return nil, fmt.Errorf("error in getting ssh connection %w", err)
+				return nil, fmt.Errorf("error in getting ssh connection for %s: %w", inst.Tags.Name, err)
 			}
 			clients[fmt.Sprintf("proxy%d", i)] = sshc
 		}
 	}
 
 	for i, instance := range output.Instances {
-		sshc, err := extAgent.NewClient(instance.PrivateIP)
+		sshc, err := extAgent.NewClient(instance.PrivateIP, config.AWSAMIUser)
 		if err != nil {
-			return nil, fmt.Errorf("error in getting ssh connection %w", err)
+			return nil, fmt.Errorf("error in getting ssh connection for %s: %w", instance.Tags.Name, err)
 		}
 		clients[fmt.Sprintf("app%d", i)] = sshc
 	}
 
 	for i, agent := range output.Agents {
-		sshc, err := extAgent.NewClient(agent.PrivateIP)
+		sshc, err := extAgent.NewClient(agent.PrivateIP, config.AWSAMIUser)
 		if err != nil {
-			return nil, fmt.Errorf("error in getting ssh connection %w", err)
+			return nil, fmt.Errorf("error in getting ssh connection for %s: %w", agent.Tags.Name, err)
 		}
 		clients[fmt.Sprintf("agent%d", i)] = sshc
 		if i == 0 {
@@ -176,7 +176,7 @@ func collect(config deployment.Config, deploymentId string, outputName string) e
 		return err
 	}
 
-	clients, err := createClients(output)
+	clients, err := createClients(config, output)
 	if err != nil {
 		return err
 	}
@@ -228,13 +228,13 @@ func collect(config deployment.Config, deploymentId string, outputName string) e
 				return sanitizedCfg, nil
 			})
 		case strings.HasPrefix(instance, "agent"):
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/ltagent.log", true, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/ltagent.log", t.Config().AWSAMIUser), true, nil)
 		case instance == "coordinator":
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/ltcoordinator.log", true, nil)
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/config/config.json", false, nil)
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/config/coordinator.json", false, nil)
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/config/simplecontroller.json", false, nil)
-			addFile(instance, "/home/ubuntu/mattermost-load-test-ng/config/simulcontroller.json", false, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/ltcoordinator.log", t.Config().AWSAMIUser), true, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/config/config.json", t.Config().AWSAMIUser), false, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/config/coordinator.json", t.Config().AWSAMIUser), false, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/config/simplecontroller.json", t.Config().AWSAMIUser), false, nil)
+			addFile(instance, fmt.Sprintf("/home/%s/mattermost-load-test-ng/config/simulcontroller.json", t.Config().AWSAMIUser), false, nil)
 			continue
 		}
 		addCmd(instance, "sudo dmesg", "dmesg.out", false, nil)

cmd/ltctl/reset.go

@@ -47,7 +47,7 @@ func RunResetCmdF(cmd *cobra.Command, args []string) error {
 	appClients := make([]*ssh.Client, len(output.Instances))
 	for i, instance := range output.Instances {
 		// BRANCH: Using private ip
-		client, err := extAgent.NewClient(instance.PrivateIP)
+		client, err := extAgent.NewClient(instance.PrivateIP, t.Config().AWSAMIUser)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}
@@ -56,7 +56,7 @@ func RunResetCmdF(cmd *cobra.Command, args []string) error {
 	}
 
 	// BRANCH: Private IP
-	agentClient, err := extAgent.NewClient(output.Agents[0].PrivateIP)
+	agentClient, err := extAgent.NewClient(output.Agents[0].PrivateIP, t.Config().AWSAMIUser)
 	if err != nil {
 		return fmt.Errorf("error in getting ssh connection %w", err)
 	}

comparison/deployment.go

@@ -62,7 +62,7 @@ func provisionFiles(t *terraform.Terraform, dpConfig *deploymentConfig, baseBuil
 	}
 	clients := make([]*ssh.Client, len(output.Instances))
 	for i, instance := range output.Instances {
-		client, err := extAgent.NewClient(instance.PrivateIP)
+		client, err := extAgent.NewClient(instance.PrivateIP, t.Config().AWSAMIUser)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}

comparison/loadtest.go

@@ -178,15 +178,15 @@ func initLoadTest(t *terraform.Terraform, buildCfg BuildConfig, dumpFilename str
 		return err
 	}
 
-	agentClient, err := extAgent.NewClient(tfOutput.Agents[0].PrivateIP)
+	agentClient, err := extAgent.NewClient(tfOutput.Agents[0].PrivateIP, t.Config().AWSAMIUser)
 	if err != nil {
 		return fmt.Errorf("error in getting ssh connection %w", err)
 	}
 	defer agentClient.Close()
 
 	appClients := make([]*ssh.Client, len(tfOutput.Instances))
 	for i, instance := range tfOutput.Instances {
-		client, err := extAgent.NewClient(instance.PrivateIP)
+		client, err := extAgent.NewClient(instance.PrivateIP, t.Config().AWSAMIUser)
 		if err != nil {
 			return fmt.Errorf("error in getting ssh connection %w", err)
 		}
@@ -203,7 +203,7 @@ func initLoadTest(t *terraform.Terraform, buildCfg BuildConfig, dumpFilename str
 	buildFileName := filepath.Base(buildCfg.URL)
 	installCmd := deployment.Cmd{
 		Msg:     "Installing app",
-		Value:   fmt.Sprintf("cd /home/ubuntu && tar xzf %s && cp /opt/mattermost/config/config.json . && sudo rm -rf /opt/mattermost && sudo mv mattermost /opt/ && mv config.json /opt/mattermost/config/", buildFileName),
+		Value:   fmt.Sprintf("cd ~ && tar xzf %s && cp /opt/mattermost/config/config.json . && sudo rm -rf /opt/mattermost && sudo mv mattermost /opt/ && mv config.json /opt/mattermost/config/", buildFileName),
 		Clients: appClients,
 	}
 

coordinator/performance/monitor.go

@@ -47,14 +47,17 @@ func NewMonitor(config MonitorConfig, log *mlog.Logger) (*Monitor, error) {
 // Run will start the performance monitoring process.
 func (m *Monitor) Run() <-chan Status {
 	go func() {
+		ticker := time.NewTicker(time.Duration(m.config.UpdateIntervalMs) * time.Millisecond)
+		defer ticker.Stop()
+
 		m.log.Info("monitor: started")
 		for {
 			m.statusChan <- m.runQueries()
 			select {
 			case <-m.stopChan:
 				m.log.Info("monitor: shutting down")
 				return
-			case <-time.After(time.Duration(m.config.UpdateIntervalMs) * time.Millisecond):
+			case <-ticker.C:
 			}
 		}
 	}()

deployment/config.go

@@ -31,6 +31,8 @@ type Config struct {
 	AWSAvailabilityZone string `default:"us-east-1c"`
 	// AWSAMI is the AMI to use for all EC2 instances.
 	AWSAMI string `default:"ami-0fa37863afb290840"`
+	// AWSAMIUser is the user to use when connecting to the AMI.
+	AWSAMIUser string `default:"ec2-user"`
 	// ClusterName is the name of the cluster.
 	ClusterName string `default:"loadtest" validate:"alpha"`
 	// ClusterVpcID is the id of the VPC associated to the resources.

deployment/opensearch/roundtripper_test.go

@@ -101,7 +101,7 @@ func setupSSHClient(t *testing.T) *ltssh.Client {
 	// Wait for the SSH server to start
 	var sshc *ltssh.Client
 	require.Eventually(t, func() bool {
-		sshc, err = extAgent.NewClientWithPort(sshIP, sshPort)
+		sshc, err = extAgent.NewClientWithPort(sshIP, sshPort, "ubuntu")
 		return err == nil
 	}, 5*time.Second, 100*time.Millisecond)
 

deployment/terraform/agent.go

@@ -18,7 +18,7 @@ import (
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 )
 
-const dstUsersFilePath = "/home/ubuntu/users.txt"
+const dstUsersFilePath = "/home/%s/users.txt"
 
 func (t *Terraform) generateLoadtestAgentConfig() (*loadtest.Config, error) {
 	cfg, err := loadtest.ReadConfig("")
@@ -39,7 +39,7 @@ func (t *Terraform) generateLoadtestAgentConfig() (*loadtest.Config, error) {
 	cfg.ConnectionConfiguration.AdminPassword = t.config.AdminPassword
 
 	if t.config.UsersFilePath != "" {
-		cfg.UsersConfiguration.UsersFilePath = dstUsersFilePath
+		cfg.UsersConfiguration.UsersFilePath = fmt.Sprintf(dstUsersFilePath, t.Config().AWSAMIUser)
 	}
 
 	return cfg, nil
@@ -103,15 +103,16 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent) error {
 		go func() {
 			defer wg.Done()
 
-			sshc, err := extAgent.NewClient(instance.PrivateIP)
+			sshc, err := extAgent.NewClient(instance.PrivateIP, t.Config().AWSAMIUser)
 			if err != nil {
 				mlog.Error("error creating ssh client", mlog.Err(err), mlog.Int("agent", agentNumber))
 				foundErr.Store(true)
 				return
 			}
+
 			mlog.Info("Configuring agent", mlog.String("ip", instance.PrivateIP), mlog.Int("agent", agentNumber))
 			if uploadBinary {
-				dstFilePath := "/home/ubuntu/tmp.tar.gz"
+				dstFilePath := fmt.Sprintf("/home/%s/tmp.tar.gz", t.Config().AWSAMIUser)
 				mlog.Info("Uploading binary", mlog.String("file", packagePath), mlog.Int("agent", agentNumber))
 				if out, err := sshc.UploadFile(packagePath, dstFilePath, false); err != nil {
 					mlog.Error("error uploading file", mlog.String("path", packagePath), mlog.String("output", string(out)), mlog.Err(err), mlog.Int("agent", agentNumber))
@@ -136,10 +137,11 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent) error {
 
 			tplVars := map[string]any{
 				"blockProfileRate": t.config.PyroscopeSettings.BlockProfileRate,
-				"execStart":        baseAPIServerCmd,
+				"execStart":        fmt.Sprintf(baseAPIServerCmd, t.Config().AWSAMIUser),
+				"User":             t.Config().AWSAMIUser,
 			}
 			if t.config.EnableAgentFullLogs {
-				tplVars["execStart"] = fmt.Sprintf("/bin/bash -c '%s &>> /home/ubuntu/ltapi.log'", baseAPIServerCmd)
+				tplVars["execStart"] = fmt.Sprintf("/bin/bash -c '%s &>> /home/%s/ltapi.log'", fmt.Sprintf(baseAPIServerCmd, t.Config().AWSAMIUser), t.Config().AWSAMIUser)
 			}
 			buf := bytes.NewBufferString("")
 			tpl.Execute(buf, tplVars)
@@ -151,16 +153,25 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent) error {
 				return
 			}
 
+			otelcolConfigFile, err := fillConfigTemplate(otelcolConfig, map[string]any{
+				"User": t.Config().AWSAMIUser,
+			})
+			if err != nil {
+				mlog.Error("unable to render otelcol config", mlog.Int("agent", agentNumber), mlog.Err(err))
+				foundErr.Store(true)
+				return
+			}
+
 			batch := []uploadInfo{
 				{srcData: strings.TrimPrefix(buf.String(), "\n"), dstPath: "/lib/systemd/system/ltapi.service", msg: "Uploading load-test api service file"},
 				{srcData: strings.TrimPrefix(clientSysctlConfig, "\n"), dstPath: "/etc/sysctl.conf"},
 				{srcData: strings.TrimPrefix(limitsConfig, "\n"), dstPath: "/etc/security/limits.conf"},
 				{srcData: strings.TrimPrefix(prometheusNodeExporterConfig, "\n"), dstPath: "/etc/default/prometheus-node-exporter"},
-				{srcData: strings.TrimSpace(otelcolConfig), dstPath: "/etc/otelcol-contrib/config.yaml"},
+				{srcData: strings.TrimSpace(otelcolConfigFile), dstPath: "/etc/otelcol-contrib/config.yaml"},
 			}
 
 			if t.config.UsersFilePath != "" {
-				batch = append(batch, uploadInfo{srcData: strings.Join(splitFiles[agentNumber], "\n"), dstPath: dstUsersFilePath, msg: "Uploading list of users credentials"})
+				batch = append(batch, uploadInfo{srcData: strings.Join(splitFiles[agentNumber], "\n"), dstPath: fmt.Sprintf(dstUsersFilePath, t.Config().AWSAMIUser), msg: "Uploading list of users credentials"})
 			}
 
 			// If SiteURL is set, update /etc/hosts to point to the correct IP
@@ -181,6 +192,10 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent) error {
 				return
 			}
 
+			if err := t.setupPrometheusNodeExporter(sshc); err != nil {
+				mlog.Error("error setting up prometheus node exporter", mlog.Err(err), mlog.Int("agent", agentNumber))
+			}
+
 			cmd = "sudo systemctl restart otelcol-contrib"
 			if out, err := sshc.RunCommand(cmd); err != nil {
 				mlog.Error("error running ssh command", mlog.Int("agent", agentNumber), mlog.String("cmd", cmd), mlog.String("out", string(out)), mlog.Err(err))
@@ -195,7 +210,7 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent) error {
 			}
 
 			mlog.Info("Starting load-test api server", mlog.Int("agent", agentNumber))
-			if out, err := sshc.RunCommand("sudo systemctl daemon-reload && sudo service ltapi restart"); err != nil {
+			if out, err := sshc.RunCommand("sudo systemctl daemon-reload && sudo systemctl restart ltapi"); err != nil {
 				mlog.Error("error starting load-test api server", mlog.String("output", string(out)), mlog.Err(err), mlog.Int("agent", agentNumber))
 				foundErr.Store(true)
 				return
@@ -217,7 +232,7 @@ func (t *Terraform) initLoadtest(extAgent *ssh.ExtAgent, initData bool) error {
 		return errors.New("there are no agents to initialize load-test")
 	}
 	ip := t.output.Agents[0].PrivateIP
-	sshc, err := extAgent.NewClient(ip)
+	sshc, err := extAgent.NewClient(ip, t.Config().AWSAMIUser)
 	if err != nil {
 		return err
 	}
@@ -230,7 +245,7 @@ func (t *Terraform) initLoadtest(extAgent *ssh.ExtAgent, initData bool) error {
 	if err != nil {
 		return err
 	}
-	dstPath := "/home/ubuntu/mattermost-load-test-ng/config/config.json"
+	dstPath := fmt.Sprintf("/home/%s/mattermost-load-test-ng/config/config.json", t.Config().AWSAMIUser)
 	mlog.Info("Uploading updated config file")
 	if out, err := sshc.Upload(bytes.NewReader(data), dstPath, false); err != nil {
 		return fmt.Errorf("error uploading file, output: %q: %w", out, err)

deployment/terraform/assets/cluster.tf

@@ -50,7 +50,7 @@ resource "aws_instance" "app_server" {
   connection {
     # The default username for our AMI
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     # BRANCH: Use private IP (using private subnet)
     host = self.private_ip
   }
@@ -75,7 +75,7 @@ resource "aws_instance" "app_server" {
 
   provisioner "file" {
     source      = var.mattermost_license_file
-    destination = "/home/ubuntu/mattermost.mattermost-license"
+    destination = "/home/${var.aws_ami_user}/mattermost.mattermost-license"
   }
 
   provisioner "remote-exec" {
@@ -148,7 +148,7 @@ resource "aws_instance" "metrics_server" {
   connection {
     # The default username for our AMI
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     # BRANCH: Use private IP (using private subnet)
     host = self.private_ip
   }
@@ -201,7 +201,7 @@ resource "aws_instance" "proxy_server" {
   connection {
     # The default username for our AMI
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     # BRANCH: Use private IP (using private subnet)
     host = self.private_ip
   }
@@ -363,7 +363,7 @@ resource "aws_instance" "loadtest_agent" {
 
   connection {
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     # BRANCH: Use private IP (using private subnet)
     host = self.private_ip
   }
@@ -709,7 +709,7 @@ resource "aws_instance" "job_server" {
   connection {
     # The default username for our AMI
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     # BRANCH: Use private IP (using private subnet)
     host = self.private_ip
   }
@@ -732,7 +732,7 @@ resource "aws_instance" "job_server" {
 
   provisioner "file" {
     source      = var.mattermost_license_file
-    destination = "/home/ubuntu/mattermost.mattermost-license"
+    destination = "/home/${var.aws_ami_user}/mattermost.mattermost-license"
   }
 
   provisioner "remote-exec" {
@@ -753,15 +753,15 @@ resource "null_resource" "s3_dump" {
 }
 
 // Keycloak
-resource "aws_instance" "keycloak" {
+resource "aws_instance" "keycloak_server" {
   tags = {
     Name = "${var.cluster_name}-keycloak"
   }
 
   connection {
     # The default username for our AMI
     type = "ssh"
-    user = "ubuntu"
+    user = var.aws_ami_user
     host = self.private_dns
   }
 

deployment/terraform/assets/datasource.yaml

@@ -8,11 +8,12 @@ datasources:
   type: prometheus
   access: proxy
   orgId: 1
-  url: %s
+  url: http://localhost:9090
   isDefault: true
   version: 1
   editable: true
   jsonData:
+    httpMethod: GET
     timeInterval: "5s"
 - name: Loki
   type: loki

deployment/terraform/assets/outputs.tf

@@ -15,7 +15,7 @@ output "metricsServer" {
 }
 
 output "keycloakServer" {
-  value = aws_instance.keycloak
+  value = aws_instance.keycloak_server
 }
 
 output "proxy" {
@@ -51,4 +51,3 @@ output "dbSecurityGroup" {
 output "jobServers" {
   value = aws_instance.job_server[*]
 }
-