롤링 방식으로 무중단 배포하기, 젠킨스-nginx 설정파일 수정을 위한 공개키 복사

롤링 방식으로 무중단 배포하기

https://gayeon-l.tistory.com/429

롤링방식 무중단배포 구현하기

 

2차 프로젝트 때 썼던 롤링 배포 스크립트를 참고했다.

 

pipeline {
    tools {
        gradle "GRADLE"
    }
    agent any
    stages {
        stage('Clone') {
            steps {
                git branch: 'main', url: 'git 레포지토리 주소'
            }
        }

        stage('Set Permissions') {
            steps {
                sh 'chmod +x ./gradlew'
            }
        }

        stage('Test') {
            steps {
                script {
                    sh './gradlew test'
                }
            }
        }

        stage('Build') {
            steps {
                sh './gradlew clean build'
                script {
                    def jarFilePattern = '.jar 경로'
                    def jarFile = sh(script: "ls $jarFilePattern | grep -v plain", returnStdout: true).trim()
                    echo "Built JAR file: $jarFile"

                    def appServerIps = ['app ip', 'app2 ip']
                    copyJarToRemote(appServerIps[0], jarFile) // Copy JAR to first server
                }
            }
        }

        stage('Deploy') {
            steps {
                script {
                    def appServerIps = ['app ip', 'app2 ip']
                    def nginxIp = 'ip'

                    // Deploy on first server
                    deployOnServer(appServerIps[0], nginxIp)

                    // Copy JAR to the second server
                    copyJarToRemote(appServerIps[1], jarFile)

                    // Deploy on second server
                    deployOnServer(appServerIps[1], nginxIp)
                }
            }
        }
    }
    post {
        success {
            echo "Build and deployment succeeded."
        }
        failure {
            echo "Build or deployment failed."
        }
    }
}

def copyJarToRemote(targetServerIp, jarFile) {
    def deployPath = '/home/ubuntu'

    sshagent(['credential key 이름']) {
        def scpCmd = "scp -o StrictHostKeyChecking=no $jarFile ubuntu@$targetServerIp:$deployPath/"
        def scpResult = sh(script: scpCmd, returnStatus: true)
        if (scpResult != 0) {
            error "Failed to copy jar file to $targetServerIp"
        } else {
            echo "Successfully copied jar file to $targetServerIp"
        }
    }
}

def deployOnServer(ip, nginxIp) {
    def deployPath = '/home/ubuntu'
    def jarFilePattern = "파일명.jar"
    def runAppCommand = "nohup java -jar $deployPath/$jarFilePattern > $deployPath/log.log 2>&1 &"
    def checkLogCommand = "grep -q 'Started' $deployPath/log.log"
    def backupJarFile = '파일명-backup.jar'
    def maxAttempts = 10
    def sleepInterval = 5

    sshagent(['jenkins-key']) {
        echo "Starting deployment on $ip"
        
        updateLoadBalancer(ip, nginxIp, 'remove')

        // 1. Stop existing process and free port
        stopExistingProcess(ip)

        // 2. Create backup if not exists
        def backupCmd = """
            ssh -o StrictHostKeyChecking=no ubuntu@$ip '
            # Find the JAR file matching the pattern
            jarFile=\$(find $deployPath -name "$jarFilePattern" | head -n 1)
            if [ -n "\$jarFile" ]; then
                # Check if the backup file already exists
                if [ ! -f $deployPath/$backupJarFile ]; then
                    # If not, copy the found JAR file to the backup
                    cp "\$jarFile" "$deployPath/$backupJarFile"
                else
                    echo "Backup file already exists on $ip"
                fi
            else
                echo "No JAR file found to back up"
                exit 1
            fi
            '
        """
        def backupResult = sh(script: backupCmd, returnStatus: true)
        if (backupResult != 0) {
            error "Failed to create backup file on $ip"
        }

        // 3. Deploy new version
        def runCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$runAppCommand'"
        def runResult = sh(script: runCmd, returnStatus: true)
        if (runResult != 0) {
            error "Failed to start application on $ip"
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 4. Check if the new version started successfully
        def attempts = 0
        def deploymentSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Deployment to $ip was successful."
                deploymentSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!deploymentSuccess) {
            error "Deployment to $ip failed. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 5. Run CD test
        def cdTestResult = checkApplicationStatus(ip)
        echo "CD test result for $ip: '${cdTestResult}'"
        if (cdTestResult != "200") {
            error "CD test failed for $ip. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        } else {
            echo "CD test passed for $ip."
        }

        // 6. Update load balancer with new server IP
        updateLoadBalancer(ip, nginxIp, 'add')

        // Wait before moving to the next server to ensure stability
        sleep 10
    }
}

def stopExistingProcess(ip) {
    // 현재 실행 중인 프로세스를 찾고 종료
    sshagent(['credential key 이름']) {
        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$ip '
            # Kill the process using the JAR file
            ps aux | grep 파일명.jar | grep -v grep | awk "{print \$2}" | xargs sudo kill -9 || true
            # Kill any process using port 8080
            sudo lsof -ti:8080 | xargs sudo kill -9 || true
            '
        """, returnStatus: true
    }
}

def checkApplicationStatus(ip) {
    def checkStatusCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080/products'"
    def statusCode = sh(script: checkStatusCmd, returnStdout: true).trim()
    return statusCode
}

def rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand) {
    def maxAttempts = 10
    def sleepInterval = 3

    sshagent(['jenkins-key']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true

        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'if [ -f $deployPath/$backupJarFile ]; then
                mv -f $deployPath/$backupJarFile $deployPath/logging-sample-prj-*-no_db.jar
                $runAppCommand
            else
                echo "Backup file not found on $targetServerIp"
                exit 1
            fi'
        """, returnStatus: true

        def attempts = 0
        def rollbackSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Rollback to previous version on $targetServerIp was successful."
                rollbackSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!rollbackSuccess) {
            error "Rollback to previous version on $targetServerIp failed."
        }
    }
}

def updateLoadBalancer(serverIp, nginxIp, action) {
    def nginxConfigPath = '/etc/nginx/nginx.conf'

    def serverCommand = action == 'remove' ?
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
# 서버 IP 제거
            sudo sed -i \\"/server $serverIp:8080/d\\" $nginxConfigPath && \\

# Nginx 구성을 테스트
            sudo nginx -t && \\

# Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """ :
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
# 서버 IP 추가
            sudo sed -i \\"/upstream nginx {/a \\
            \\            server $serverIp:8080 weight=100 max_fails=3 fail_timeout=3s;\\" $nginxConfigPath && \\

# Nginx 구성을 테스트
            sudo nginx -t && \\

# Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """

    def result = sh(script: serverCommand, returnStatus: true)
    if (result != 0) {
        error "Failed to ${action == 'remove' ? 'remove' : 'add'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    } else {
        echo "Successfully ${action == 'remove' ? 'removed' : 'added'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    }
}

 

 

---

 

젠킨스-nginx 설정파일 수정을 위한 공개키 복사

 

 

그런데 permission denied가 떴다.

이 에러는 젠킨스가 설치된 인스턴스의 rsa 공개키가 nginx 인스턴스에 없어서 ssh 접속을 못해서 생긴다.

 

 

 

그래서 젠킨스의 rsa 공개키를 복사해서 nginx의 authorized_keys에 붙여넣었다.

 

 

 

키를 추가하니 설정 파일 접근, 수정에 성공했다.

 

 

 

그런데 이번에는 빌드 또는 배포 단계에서 실패했다고 출력되었다.

하지만 배포 후 CD 테스트까지 한 뒤에 로드밸런서에 ip를 추가하기 때문에

빌드에 실패했다면 배포도 불가능했을 것이고 배포에 실패했다면 로드밸런서에 ip를 추가하는 단계까지 가지 않았을 것이다.

MissingPropertyException 에러를 찾아보니 젠킨스 인스턴스에 jar 파일이 있는 경로인 jarFile 변수가 정의되지 않았거나 올바르게 접근할 수 없는 범위에서 참조될 때 발생한다고 했다.

 

그래서 스크립트를 다시 보니 build 단계에서 jarFile을 정의하고 Deploy에서 쓰려고 해서 정상적으로 참조하지 못했던 것 같다.

그래서 jar 파일의 경로를 환경변수로 저장해 다른 스크립트에서 참조할 수 있도록 수정했다.

수정한 스크립트

pipeline {
    tools {
        gradle "GRADLE"
    }
    agent any
    stages {
        stage('Clone') {
            steps {
                git branch: 'main', url: 'git 레포지토리 주소'
            }
        }

        stage('Set Permissions') {
            steps {
                sh 'chmod +x ./gradlew'
            }
        }

        stage('Test') {
            steps {
                script {
                    sh './gradlew test'
                }
            }
        }

        stage('Build') {
            steps {
                script {
                    def jarFilePattern = 'jar 경로'
                    def jarFile = sh(script: "ls $jarFilePattern | grep -v plain", returnStdout: true).trim()
                    echo "Built JAR file: $jarFile"
                    // Save the jarFile path to environment variable
                    env.JAR_FILE = jarFile
                }
            }
        }

        stage('Deploy') {
            steps {
                script {
                    def appServerIps = ['app ip', 'app2 ip']
                    def nginxIp = 'ip'

                    // Deploy on first server
                    deployOnServer(appServerIps[0], nginxIp)

                    // Copy JAR to the second server
                    copyJarToRemote(appServerIps[1], env.JAR_FILE)

                    // Deploy on second server
                    deployOnServer(appServerIps[1], nginxIp)
                }
            }
        }
    }
    post {
        success {
            echo "Build and deployment succeeded."
        }
        failure {
            echo "Build or deployment failed."
        }
    }
}

def copyJarToRemote(targetServerIp, jarFile) {
    def deployPath = '/home/ubuntu'

    sshagent(['credential key 이름']) {
        def scpCmd = "scp -o StrictHostKeyChecking=no $jarFile ubuntu@$targetServerIp:$deployPath/"
        def scpResult = sh(script: scpCmd, returnStatus: true)
        if (scpResult != 0) {
            error "Failed to copy jar file to $targetServerIp"
        } else {
            echo "Successfully copied jar file to $targetServerIp"
        }
    }
}

def deployOnServer(ip, nginxIp) {
    def deployPath = '/home/ubuntu'
    def jarFilePattern = "파일명.jar"
    def runAppCommand = "nohup java -jar $deployPath/$jarFilePattern > $deployPath/log.log 2>&1 &"
    def checkLogCommand = "grep -q 'Started' $deployPath/log.log"
    def backupJarFile = '파일명-backup.jar'
    def maxAttempts = 10
    def sleepInterval = 5

    sshagent(['credential key 이름']) {
        echo "Starting deployment on $ip"

		updateLoadBalancer(ip, nginxIp, 'remove')
	
        // 1. Stop existing process and free port
        stopExistingProcess(ip)

        // 2. Create backup if not exists
        def backupCmd = """
            ssh -o StrictHostKeyChecking=no ubuntu@$ip '
            # Find the JAR file matching the pattern
            jarFile=\$(find $deployPath -name "$jarFilePattern" | head -n 1)
            if [ -n "\$jarFile" ]; then
                # Check if the backup file already exists
                if [ ! -f $deployPath/$backupJarFile ]; then
                    # If not, copy the found JAR file to the backup
                    cp "\$jarFile" "$deployPath/$backupJarFile"
                else
                    echo "Backup file already exists on $ip"
                fi
            else
                echo "No JAR file found to back up"
                exit 1
            fi
            '
        """
        def backupResult = sh(script: backupCmd, returnStatus: true)
        if (backupResult != 0) {
            error "Failed to create backup file on $ip"
        }

        // 3. Deploy new version
        def runCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$runAppCommand'"
        def runResult = sh(script: runCmd, returnStatus: true)
        if (runResult != 0) {
            error "Failed to start application on $ip"
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 4. Check if the new version started successfully
        def attempts = 0
        def deploymentSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Deployment to $ip was successful."
                deploymentSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!deploymentSuccess) {
            error "Deployment to $ip failed. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 5. Run CD test
        def cdTestResult = checkApplicationStatus(ip)
        echo "CD test result for $ip: '${cdTestResult}'"
        if (cdTestResult != "200") {
            error "CD test failed for $ip. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        } else {
            echo "CD test passed for $ip."
        }

        // 6. Update load balancer with new server IP
        updateLoadBalancer(ip, nginxIp, 'add')

        // Wait before moving to the next server to ensure stability
        sleep 10
    }
}

def stopExistingProcess(ip) {
    // 현재 실행 중인 프로세스를 찾고 종료
    sshagent(['credential key 이름']) {
        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$ip '
            # Kill the process using the JAR file
            ps aux | grep 파일명.jar | grep -v grep | awk "{print \$2}" | xargs sudo kill -9 || true
            # Kill any process using port 8080
            sudo lsof -ti:8080 | xargs sudo kill -9 || true
            '
        """, returnStatus: true
    }
}

def checkApplicationStatus(ip) {
    def checkStatusCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080/products'"
    def statusCode = sh(script: checkStatusCmd, returnStdout: true).trim()
    return statusCode
}

def rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand) {
    def maxAttempts = 10
    def sleepInterval = 3

    sshagent(['jenkins-key']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true

        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'if [ -f $deployPath/$backupJarFile ]; then
                mv -f $deployPath/$backupJarFile $deployPath/파일명.jar
                $runAppCommand
            else
                echo "Backup file not found on $targetServerIp"
                exit 1
            fi'
        """, returnStatus: true

        def attempts = 0
        def rollbackSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Rollback to previous version on $targetServerIp was successful."
                rollbackSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!rollbackSuccess) {
            error "Rollback to previous version on $targetServerIp failed."
        }
    }
}

def updateLoadBalancer(serverIp, nginxIp, action) {
    def nginxConfigPath = '/etc/nginx/nginx.conf'

    def serverCommand = action == 'remove' ?
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
# 서버 IP 제거
            sudo sed -i \\"/server $serverIp:8080/d\\" $nginxConfigPath && \\

# Nginx 구성을 테스트
            sudo nginx -t && \\

# Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """ :
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
# 서버 IP 추가
            sudo sed -i \\"/upstream nginx {/a \\
            \\            server $serverIp:8080 weight=100 max_fails=3 fail_timeout=3s;\\" $nginxConfigPath && \\

# Nginx 구성을 테스트
            sudo nginx -t && \\

# Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """

    def result = sh(script: serverCommand, returnStatus: true)
    if (result != 0) {
        error "Failed to ${action == 'remove' ? 'remove' : 'add'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    } else {
        echo "Successfully ${action == 'remove' ? 'removed' : 'added'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
    }
}

 

 

다행히 빌드 및 배포가 정상적으로 동작했다.

 

 

 

nginx의 설정파일에도 두 배포 서버의 정보가 재등록된 것을 볼 수 있었다.