blue/green 방식 무중단배포 구현하기

blue/green 방식 구현하기

 

블루: 구버전

그린: 신버전

 

장점

• 구버전의 인스턴스가 그대로 남아있어 롤백이 비교적 쉽다.

 

원래의 signup 페이지에 있는 회원가입 글자를

-무중단배포(블루/그린)-를 추가해서 무중단 배포가 되고 있는지 확인할 것이다.

 

 

 

서비스가 끊기지 않도록 새 버전으로 배포되는 중에는 Nginx 로드 밸런서 설정에서

업데이트 중인 서버 아이피를 삭제했다가 업데이트가 완료되면 다시 넣도록 할 것이다.

젠킨스에서 nginx 설정 파일을 수정하기 위해 젠킨스의 공개키를 nginx에 저장했다.

 

 

 

저장한 뒤

SSH 인증을 위해 공개 키 기반 인증을 사용하려면

위 파일의 맨 아래에 PubkeyAuthentication yes를 입력하고

 

 

 

서비스를 재시작한다.

 

 

 

app01-1에만 새로운 버전이 배포되었다.

 

 

 

배포되는 과정에서 설정파일을 cat으로 출력해보면

 

 

 

업데이트 전에는 지웠다가

 

 

 

업데이트가 되면 다시 추가되는 모습을 볼 수 있었다.

 

 

 

 

blue/green 코드

pipeline {
    tools {
        gradle "GRADLE"
    }
    agent any
    stages {
        stage('Clone') {
            steps {
                git branch: 'master', url: 'git 레포지토리 주소'
            }
        }

        stage('Set Permissions') {
            steps {
                sh 'chmod +x ./gradlew'
            }
        }

        stage('Test') {
            steps {
                script {
                    sh './gradlew test'
                }
            }
        }

        stage('Build') {
            steps {
                sh './gradlew clean build'
                sh './gradlew clean build -Pdb.url=jdbc:mysql://ip:포트번호/데이터베이스명?useSSL=false -Pdb.username=유저명 -Pdb.password=비밀번호 -Pdb.driver=com.mysql.cj.jdbc.Driver'
                script {
                    copyJarToRemote('app01 ip')
                    copyJarToRemote('app01-1 ip')
                }
            }
        }

        stage('Deploy') {
            steps {
                script {
                    def appServerIp = 'app01 ip'
                    def standbyServerIp = 'app01-1 ip'
                    def nginxIp = 'nginx ip' // Nginx 인스턴스 IP 주소 설정
                    deployAndTest(appServerIp, standbyServerIp, nginxIp)
                }
            }
        }
    }
    post {
        success {
            echo "Build and deployment succeeded."
        }
        failure {
            echo "Build or deployment failed."
        }
    }
}

def copyJarToRemote(targetServerIp) {
    def jarFile = 'jar 경로'
    def deployPath = '/home/ubuntu'

    sshagent(['credential key 이름']) {
        def scpCmd = "scp -o StrictHostKeyChecking=no $jarFile ubuntu@$targetServerIp:$deployPath/"
        def scpResult = sh(script: scpCmd, returnStatus: true)
        if (scpResult != 0) {
            echo "Failed to copy jar file to $targetServerIp"
            error "Failed to copy jar file"
        } else {
            echo "Successfully copied jar file to $targetServerIp"
        }
    }
}

def deployAndTest(appServerIp, standbyServerIp, nginxIp) {
    def jarFile = '/home/ubuntu/파일명.jar'
    def deployPath = '/home/ubuntu'
    def runAppCommand = "nohup java -jar $jarFile > $deployPath/log.log 2>&1 &"
    def checkLogCommand = "grep -q 'Started' $deployPath/log.log"
    def maxAttempts = 10
    def sleepInterval = 3
    def backupJarFile = '파일명-backup.jar'

    sshagent(['credential key 이름']) {
        // Stopping existing process on standby server
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$standbyServerIp 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true

        // Creating backup if not exists
        def checkBackupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$standbyServerIp '[ ! -f $deployPath/$backupJarFile ]'"
        def checkBackupResult = sh(script: checkBackupCmd, returnStatus: true)
        if (checkBackupResult == 0) {
            def backupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$standbyServerIp 'cp $deployPath/파일명.jar $deployPath/$backupJarFile'"
            def backupResult = sh(script: backupCmd, returnStatus: true)
            if (backupResult != 0) {
                echo "Failed to create backup file on $standbyServerIp"
                error "Failed to create backup file"
            }
        } else {
            echo "Backup file already exists on $standbyServerIp"
        }

        // Remove standby server from upstream
        def nginxConfigPath = '/etc/nginx/nginx.conf'
        def removeStandbyServerCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'sudo sed -i \"/server $standbyServerIp:8080/d\" $nginxConfigPath && sudo systemctl reload nginx'"
        def removeStandbyResult = sh(script: removeStandbyServerCmd, returnStatus: true)
        if (removeStandbyResult != 0) {
            echo "Failed to remove $standbyServerIp from load balancer"
            error "Failed to remove $standbyServerIp from load balancer"
        } else {
            echo "Successfully removed $standbyServerIp from load balancer"
        }

        // Deploying new version on standby server
        def runCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$standbyServerIp '$runAppCommand'"
        def runResult = sh(script: runCmd, returnStatus: true)
        if (runResult != 0) {
            echo "Failed to start application on $standbyServerIp"
            error "Failed to start application"
        }

        // Checking if the new version started successfully
        def attempts = 0
        def deploymentSuccess = false
        while (attempts < maxAttempts) {
            int result = sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$standbyServerIp '$checkLogCommand'", returnStatus: true
            if (result == 0) {
                echo "Deployment to $standbyServerIp was successful."
                deploymentSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!deploymentSuccess) {
            echo "Deployment to $standbyServerIp failed. Rolling back to previous version."
            rollbackToPreviousVersion(standbyServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand)
            error "Deployment to $standbyServerIp failed."
        }

        // Re-adding standby server to upstream
        def addStandbyServerCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'sudo sed -i \"/upstream cpu-bound-app/a\\    server $standbyServerIp:8080 weight=100 max_fails=3 fail_timeout=3s;\" $nginxConfigPath && sudo systemctl reload nginx'"
        def addStandbyResult = sh(script: addStandbyServerCmd, returnStatus: true)
        if (addStandbyResult != 0) {
            echo "Failed to re-add $standbyServerIp to load balancer"
            error "Failed to re-add $standbyServerIp to load balancer"
        } else {
            echo "Successfully re-added $standbyServerIp to load balancer"
        }

        // Running CD test
        def cdTestResult = sh script: "curl -s -o /dev/null -w '%{http_code}' http://$standbyServerIp:8080", returnStdout: true
        echo "CD test result: '${cdTestResult.trim()}'"
        if (cdTestResult.trim() != "200") {
            echo "CD test failed for $standbyServerIp. Rolling back to previous version."
            rollbackToPreviousVersion(standbyServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        } else {
            echo "CD test passed for $standbyServerIp."
        }
    }
}

def rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand) {
    def maxAttempts = 10
    def sleepInterval = 3

    sshagent(['credential key 이름']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true

        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'if [ -f $deployPath/$backupJarFile ]; then
                mv -f $deployPath/$backupJarFile $deployPath/파일명.jar
                $runAppCommand
            else
                echo "Backup file not found on $targetServerIp"
                exit 1
            fi'
        """, returnStatus: true

        def attempts = 0
        def rollbackSuccess = false
        while (attempts < maxAttempts) {
            int result = sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp '$checkLogCommand'", returnStatus: true
            if (result == 0) {
                echo "Rollback to previous version on $targetServerIp was successful."
                rollbackSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!rollbackSuccess) {
            echo "Rollback to previous version on $targetServerIp failed."
            error "Rollback failed."
        }
    }
}

 

 

 

+) 문제점

현재 app01-1과 app01만 있기 때문에

구버전, 신버전이 모두 로드밸런싱되어

새로고침할 때마다 구버전, 신버전이 같이 나오게 된다.

 

 

코드 수정 및 구현 완료

 

그린 = app01(72)

블루 = app01-1(26)

 

 

회원가입 --블루/그린 -- 에서

update 문자열을 추가할 것이다.

 

블루/그린 로직은 다음과 같다.

1) 그린서버 업데이트 전 nginx 설정파일에서 그린 서버의 아이피 제거

2) 그린서버 업데이트 동안에는 블루 서버만 응답 가능

3) 그린서버 업데이트 완료 후 nginx 설정파일에 다시 서버 아이피 추가 및 블루 서버 아이피 제거

 

이렇게 하면 업데이트 중에도 서비스는 중단되지 않는다.

 

 

 

그린 서버 업데이트 전에는 두 서버 모두 응답하다가

 

 

nginx 설정파일에 두 서버 모두 존재

 

 

 

그린 서버 업데이트 들어가기 전에는 블루 서버만 남겨놓는다.

 

 

 

그린 서버 업데이트 후에는 블루서버 제거, 그린서버만 남겨놓는다.

 

 

 

 

최종 blue/green 코드

pipeline {
    tools {
        gradle "GRADLE"
    }
    agent any
    stages {
        stage('Clone') {
            steps {
                git branch: 'master', url: 'git 레포지토리 주소'
            }
        }

        stage('Set Permissions') {
            steps {
                sh 'chmod +x ./gradlew'
            }
        }

        stage('Test') {
            steps {
                script {
                    sh './gradlew test'
                }
            }
        }

        stage('Build') {
            steps {
                script {
                    sh './gradlew clean build'
                    sh """
                        ./gradlew clean build \
                        -Pdb.url=jdbc:mysql://ip:포트번호/데이터베이스명?useSSL=false \
                        -Pdb.username=유저명 \
                        -Pdb.password=비밀번호 \
                        -Pdb.driver=com.mysql.cj.jdbc.Driver
                    """
                    def greenServerIp = 'app01 ip'
                    copyJarToRemote(greenServerIp) // Copy JAR to Green server
                }
            }
        }

        stage('Deploy') {
            steps {
                script {
                    def greenServerIp = 'app01 ip'
                    def blueServerIp = 'app01-1 ip'
                    def nginxIp = 'nginx ip'

                    sshagent(['credential key 이름']) {
                        // 1. Remove Green Server from Load Balancer
                        updateLoadBalancer(greenServerIp, nginxIp, 'remove')

                        // 2. Deploy to Green Server
                        deployOnServer(greenServerIp, nginxIp)

                        // 3. Access Blue Server (no change, just ensure it's running)
                        def blueStatus = checkApplicationStatus(blueServerIp)
                        echo "Blue Server status: ${blueStatus}"

                        // 4. Update Nginx configuration for Green Server
                        updateLoadBalancer(greenServerIp, nginxIp, 'add')

                        // 5. Remove Blue Server from Load Balancer after Green is confirmed
                        if (blueStatus == "200") {
                            updateLoadBalancer(blueServerIp, nginxIp, 'remove')
                        } else {
                            echo "Blue server is not responding as expected. Not removing it from the load balancer."
                        }
                    }
                }
            }
        }
    }
    post {
        success {
            echo "Build and deployment succeeded."
        }
        failure {
            echo "Build or deployment failed."
        }
    }
}

def copyJarToRemote(targetServerIp) {
    def jarFile = '.jar 파일 경로'
    def deployPath = '/home/ubuntu'

    sshagent(['JENKINS-KEY']) {
        def scpCmd = "scp -o StrictHostKeyChecking=no $jarFile ubuntu@$targetServerIp:$deployPath/"
        def scpResult = sh(script: scpCmd, returnStatus: true)
        if (scpResult != 0) {
            error "Failed to copy jar file to $targetServerIp"
        } else {
            echo "Successfully copied jar file to $targetServerIp"
        }
    }
}

def deployOnServer(ip, nginxIp) {
    def jarFile = '/home/ubuntu/파일명.jar'
    def deployPath = '/home/ubuntu'
    def runAppCommand = "nohup java -jar $jarFile > $deployPath/log.log 2>&1 &"
    def checkLogCommand = "grep -q 'Started' $deployPath/log.log"
    def backupJarFile = '파일명-backup.jar'
    def maxAttempts = 10
    def sleepInterval = 5

    sshagent(['credential key 이름']) {
        echo "Starting deployment on $ip"

        // 1. Stop existing process and free port
        stopExistingProcess(ip)

        // 2. Create backup if not exists
        def checkBackupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '[ ! -f $deployPath/$backupJarFile ]'"
        def checkBackupResult = sh(script: checkBackupCmd, returnStatus: true)
        if (checkBackupResult == 0) {
            def backupCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'cp $deployPath/파일명.jar $deployPath/$backupJarFile'"
            def backupResult = sh(script: backupCmd, returnStatus: true)
            if (backupResult != 0) {
                error "Failed to create backup file on $ip"
            }
        } else {
            echo "Backup file already exists on $ip"
        }

        // 3. Deploy new version
        def runCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$runAppCommand'"
        def runResult = sh(script: runCmd, returnStatus: true)
        if (runResult != 0) {
            error "Failed to start application on $ip"
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 4. Check if the new version started successfully
        def attempts = 0
        def deploymentSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Deployment to $ip was successful."
                deploymentSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!deploymentSuccess) {
            error "Deployment to $ip failed. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        }

        // 5. Run CD test
        def cdTestResult = checkApplicationStatus(ip)
        echo "CD test result for $ip: '${cdTestResult}'"
        if (cdTestResult != "200") {
            error "CD test failed for $ip. Rolling back to previous version."
            rollbackToPreviousVersion(ip, backupJarFile, deployPath, runAppCommand, checkLogCommand)
        } else {
            echo "CD test passed for $ip."
        }
    }
}

def checkApplicationStatus(ip) {
    def checkStatusCmd = "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'curl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080'"
    def statusCode = sh(script: checkStatusCmd, returnStdout: true).trim()
    return statusCode
}

def stopExistingProcess(ip) {
    sshagent(['credential key 이름']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$ip 'sudo lsof -ti:8080 | xargs sudo kill -9'", returnStatus: true
    }
}

def rollbackToPreviousVersion(targetServerIp, backupJarFile, deployPath, runAppCommand, checkLogCommand) {
    def maxAttempts = 10
    def sleepInterval = 3

    sshagent(['credential key 이름']) {
        sh script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'pgrep -f 파일명.jar && pkill -f 파일명.jar || echo \"No process found\"'", returnStatus: true

        sh script: """
            ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp 'if [ -f $deployPath/$backupJarFile ]; then
                mv -f $deployPath/$backupJarFile $deployPath/파일명.jar
                $runAppCommand
            else
                echo "Backup file not found on $targetServerIp"
                exit 1
            fi'
        """, returnStatus: true

        def attempts = 0
        def rollbackSuccess = false
        while (attempts < maxAttempts) {
            int result = sh(script: "ssh -o StrictHostKeyChecking=no ubuntu@$targetServerIp '$checkLogCommand'", returnStatus: true)
            if (result == 0) {
                echo "Rollback to previous version on $targetServerIp was successful."
                rollbackSuccess = true
                break
            }
            attempts++
            sleep sleepInterval
        }

        if (!rollbackSuccess) {
            error "Rollback to previous version on $targetServerIp failed."
        }
    }
}

def updateLoadBalancer(serverIp, nginxIp, action) {
    def nginxConfigPath = '/etc/nginx/nginx.conf'

    def serverCommand = action == 'remove' ?
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
            # 서버 IP 제거
            sudo sed -i \\"/server $serverIp:8080/d\\" $nginxConfigPath && \\
            
            # Nginx 구성을 테스트
            sudo nginx -t && \\
            
            # Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """ :
        """
        ssh -o StrictHostKeyChecking=no ubuntu@$nginxIp 'bash -c "
            # 서버 IP 추가
            sudo sed -i \\"/upstream cpu-bound-app {/a \\
            \\            server $serverIp:8080 weight=100 max_fails=3 fail_timeout=3s;\\" $nginxConfigPath && \\
            
            # Nginx 구성을 테스트
            sudo nginx -t && \\
            
            # Nginx를 재로드
            sudo systemctl reload nginx
        "'
        """

    sshagent(['credential key 이름']) {
        def result = sh(script: serverCommand, returnStatus: true)
        if (result != 0) {
            error "Failed to ${action == 'remove' ? 'remove' : 'add'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
        } else {
            echo "Successfully ${action == 'remove' ? 'removed' : 'added'} $serverIp ${action == 'remove' ? 'from' : 'to'} load balancer"
        }
    }
}