From 81735d045d96c7f435da36016486cfad550e72a1 Mon Sep 17 00:00:00 2001 From: Kasra Bigdeli Date: Tue, 19 Feb 2019 22:13:47 -0800 Subject: [PATCH] Backup phase1 done, WIP --- src/docker/DockerApi.ts | 16 ++-- src/docker/DockerUtils.ts | 104 ++++++++++++++++++++ src/models/ServerDockerInfo.ts | 2 +- src/routes/system/SystemRouter.ts | 5 +- src/user/system/BackupManager.ts | 151 +++++++++++++++++++++++++++--- src/user/system/CaptainManager.ts | 98 ------------------- src/utils/CaptainInstaller.ts | 2 +- src/utils/Utils.ts | 8 ++ 8 files changed, 266 insertions(+), 120 deletions(-) create mode 100644 src/docker/DockerUtils.ts diff --git a/src/docker/DockerApi.ts b/src/docker/DockerApi.ts index e4fe328..6da20e6 100644 --- a/src/docker/DockerApi.ts +++ b/src/docker/DockerApi.ts @@ -1021,13 +1021,17 @@ class DockerApi { return this.checkIfSecretExist(secretKey).then(function(secretExists) { if (secretExists) { - return true + return } else { - return self.dockerode.createSecret({ - Name: secretKey, - Labels: {}, - Data: Base64.encode(valueIfNotExist), - }) + return self.dockerode + .createSecret({ + Name: secretKey, + Labels: {}, + Data: Base64.encode(valueIfNotExist), + }) + .then(function() { + return + }) } }) } diff --git a/src/docker/DockerUtils.ts b/src/docker/DockerUtils.ts new file mode 100644 index 0000000..6ff2272 --- /dev/null +++ b/src/docker/DockerUtils.ts @@ -0,0 +1,104 @@ +import DockerApi from './DockerApi' +import SshClientImport = require('ssh2') +import ApiStatusCodes = require('../api/ApiStatusCodes') +import CaptainConstants = require('../utils/CaptainConstants') +import Logger = require('../utils/Logger') +const SshClient = SshClientImport.Client + +export default class DockerUtils { + static joinDockerNode( + dockerApi: DockerApi, + captainIpAddress: string, + isManager: boolean, + remoteNodeIpAddress: string, + privateKey: string + ) { + const remoteUserName = 'root' // Docker requires root access. It has to be root. + + return Promise.resolve() + .then(function() { + return dockerApi.getJoinToken(isManager) + }) + .then(function(token) { + return new Promise(function(resolve, reject) { + const conn = new SshClient() + conn.on('error', function(err) { + Logger.e(err) + reject( + ApiStatusCodes.createError( + ApiStatusCodes.STATUS_ERROR_GENERIC, + 'SSH Connection error!!' + ) + ) + }) + .on('ready', function() { + Logger.d('SSH Client :: ready') + conn.exec( + CaptainConstants.disableFirewallCommand + + ' ' + + dockerApi.createJoinCommand( + captainIpAddress, + token + ), + function(err, stream) { + if (err) { + Logger.e(err) + reject( + ApiStatusCodes.createError( + ApiStatusCodes.STATUS_ERROR_GENERIC, + 'SSH Running command failed!!' + ) + ) + return + } + + let hasExisted = false + + stream + .on('close', function( + code: string, + signal: string + ) { + Logger.d( + 'Stream :: close :: code: ' + + code + + ', signal: ' + + signal + ) + conn.end() + if (hasExisted) { + return + } + hasExisted = true + resolve() + }) + .on('data', function(data: string) { + Logger.d('STDOUT: ' + data) + }) + .stderr.on('data', function(data) { + Logger.e('STDERR: ' + data) + if (hasExisted) { + return + } + hasExisted = true + reject( + ApiStatusCodes.createError( + ApiStatusCodes.STATUS_ERROR_GENERIC, + 'Error during setup: ' + + data + ) + ) + }) + } + ) + }) + .connect({ + host: remoteNodeIpAddress, + port: 22, + username: remoteUserName, + privateKey: privateKey, + }) + }) + }) + } +} diff --git a/src/models/ServerDockerInfo.ts b/src/models/ServerDockerInfo.ts index 748712a..6c38336 100644 --- a/src/models/ServerDockerInfo.ts +++ b/src/models/ServerDockerInfo.ts @@ -1,6 +1,6 @@ interface ServerDockerInfo { nodeId: string - type: string + type: 'manager' | 'worker' isLeader: boolean hostname: string architecture: string diff --git a/src/routes/system/SystemRouter.ts b/src/routes/system/SystemRouter.ts index 9134957..68182be 100644 --- a/src/routes/system/SystemRouter.ts +++ b/src/routes/system/SystemRouter.ts @@ -9,6 +9,8 @@ import CaptainConstants = require('../../utils/CaptainConstants') import InjectionExtractor = require('../../injection/InjectionExtractor') import Utils from '../../utils/Utils' import * as path from 'path' +import DockerUtils from '../../docker/DockerUtils' +import DockerApi from '../../docker/DockerApi' const router = express.Router() @@ -370,7 +372,8 @@ router.post('/nodes/', function(req, res, next) { } }) .then(function() { - return CaptainManager.get().joinDockerNode( + return DockerUtils.joinDockerNode( + DockerApi.get(), captainIpAddress, isManager, remoteNodeIpAddress, diff --git a/src/user/system/BackupManager.ts b/src/user/system/BackupManager.ts index 6db46fd..ca30f79 100644 --- a/src/user/system/BackupManager.ts +++ b/src/user/system/BackupManager.ts @@ -19,6 +19,7 @@ import * as tar from 'tar' import * as fs from 'fs-extra' import Utils from '../../utils/Utils' import { BackupMeta, RestoringInfo } from '../../models/BackupMeta' +import DockerUtils from '../../docker/DockerUtils' const SshClient = SshClientImport.Client const CURRENT_NODE_DONT_CHANGE = 'CURRENT_NODE_DONT_CHANGE' @@ -27,7 +28,7 @@ const IP_PLACEHOLDER = 'replace-me-with-new-ip-or-empty-see-docs' const BACKUP_JSON = 'backup.json' const RESTORE_INSTRUCTIONS = 'restore-instructions.json' -const RESTORE_JSON_ABS_PATH = +const RESTORE_INSTRUCTIONS_ABS_PATH = CaptainConstants.restoreDirectoryPath + '/' + RESTORE_INSTRUCTIONS export interface IBackupCallbacks { @@ -62,12 +63,135 @@ export default class BackupManager { return !!this.longOperationInProgress } - startRestorationIfNeededPhase1() { + startRestorationIfNeededPhase1(captainIpAddress: string) { // if (/captain/restore/restore-instructions.json does exist): // - Connect all extra nodes via SSH and get their NodeID // - Replace the nodeId in apps with the new nodeId based on restore-instructions.json // - Create a captain-salt secret using the data in restore // - Copy restore files to proper places + const self = this + + const oldNodeIdToNewIpMap: IHashMapGeneric = {} + + return Promise.resolve() + .then(function() { + if (!fs.pathExistsSync(RESTORE_INSTRUCTIONS_ABS_PATH)) return + + return fs + .readJson(RESTORE_INSTRUCTIONS_ABS_PATH) + .then(function(restoringInfo: RestoringInfo) { + const ps: (() => Promise)[] = [] + restoringInfo.nodesMapping.forEach(n => { + let isManager = false + + restoringInfo.oldNodesForReference.forEach(oldN => { + if (oldN.nodeData.ip === n.oldIp) { + oldNodeIdToNewIpMap[oldN.nodeData.nodeId] = + n.newIp + if (oldN.nodeData.type === 'manager') { + isManager = true + } + } + }) + + ps.push(function() { + return Promise.resolve().then(function() { + return DockerUtils.joinDockerNode( + DockerApi.get(), + captainIpAddress, + isManager, + n.newIp, + fs.readFileSync( + n.privateKeyPath, + 'utf8' + ) + ) + }) + }) + }) + + return self.runPromises(ps) + }) + .then(function() { + Logger.d( + 'Waiting for 5 seconds for things to settle...' + ) + return Utils.getDelayedPromise(5000) + }) + .then(function() { + return DockerApi.get().getNodesInfo() + }) + .then(function(nodesInfo) { + function getNewNodeIdForIp(ip: string) { + let nodeId = '' + nodesInfo.forEach(n => { + if (n.ip === ip) nodeId = n.nodeId + }) + + if (nodeId) return nodeId + + throw new Error('No NodeID found for ' + ip) + } + + const configFilePathRestoring = + CaptainConstants.restoreDirectoryPath + + '/data/config-captain.json' + const configData: { + appDefinitions: IHashMapGeneric + } = fs.readJsonSync(configFilePathRestoring) + + Object.keys(oldNodeIdToNewIpMap).forEach(oldNodeId => { + const newIp = oldNodeIdToNewIpMap[oldNodeId] + Object.keys(configData.appDefinitions).forEach( + appName => { + if ( + configData.appDefinitions[appName] + .nodeId === oldNodeId + ) { + configData.appDefinitions[ + appName + ].nodeId = newIp + ? getNewNodeIdForIp(newIp) + : '' // If user removed new IP, it will mean that the user is okay with this node being automatically assigned to a node ID + } + } + ) + }) + + return fs.outputJson( + configFilePathRestoring, + configData + ) + }) + .then(function() { + return fs.readJson( + CaptainConstants.restoreDirectoryPath + + '/meta/backup.json' + ) + }) + .then(function(data: BackupMeta) { + const salt = data.salt + + if (!salt) + throw new Error( + 'Something is wrong! Salt is empty in restoring meta file' + ) + + return DockerApi.get().ensureSecret( + CaptainConstants.captainSaltSecretKey, + salt + ) + }) + .then(function() { + fs.move( + CaptainConstants.restoreDirectoryPath + '/data', + CaptainConstants.captainDataDirectory + ) + }) + }) + .then(function() { + Logger.d('Restoration Phase#1 is completed!') + }) } startRestorationIfNeededPhase2() { @@ -92,9 +216,9 @@ export default class BackupManager { // 1) /captain/restore/restore-instructions.json exists // 2) Or, /captain/restore does not exist - if (fs.pathExistsSync(RESTORE_JSON_ABS_PATH)) { + if (fs.pathExistsSync(RESTORE_INSTRUCTIONS_ABS_PATH)) { return self.processRestoreInstructions( - fs.readJsonSync(RESTORE_JSON_ABS_PATH) + fs.readJsonSync(RESTORE_INSTRUCTIONS_ABS_PATH) ) } }) @@ -148,7 +272,7 @@ export default class BackupManager { if (!!n.newIp) { if (n.newIp === IP_PLACEHOLDER) { throw new Error( - `See backup docs! You must replace the place holder: ${IP_PLACEHOLDER} in ${RESTORE_JSON_ABS_PATH}` + `See backup docs! You must replace the place holder: ${IP_PLACEHOLDER} in ${RESTORE_INSTRUCTIONS_ABS_PATH}` ) } @@ -172,17 +296,18 @@ export default class BackupManager { } }) - return self.runPromises(connectingFuncs, 0) + return self.runPromises(connectingFuncs) } runPromises( promises: (() => Promise)[], - curr: number + curr?: number ): Promise { + let currCorrected = curr ? curr : 0 const self = this - if (promises.length > curr) { - return promises[curr]().then(function() { - return self.runPromises(promises, curr + 1) + if (promises.length > currCorrected) { + return promises[currCorrected]().then(function() { + return self.runPromises(promises, currCorrected + 1) }) } @@ -313,7 +438,7 @@ export default class BackupManager { if (!fs.statSync(dirPath).isDirectory()) throw new Error('restore directory is not a directory!!') - if (!fs.pathExistsSync(RESTORE_JSON_ABS_PATH)) { + if (!fs.pathExistsSync(RESTORE_INSTRUCTIONS_ABS_PATH)) { return Promise.resolve() // .then(function() { const metaData = fs.readJsonSync( @@ -328,7 +453,7 @@ export default class BackupManager { ) return fs.outputJson( - RESTORE_JSON_ABS_PATH, + RESTORE_INSTRUCTIONS_ABS_PATH, self.createRestoreInstructionData( metaData, configData @@ -337,7 +462,7 @@ export default class BackupManager { }) } - if (!fs.statSync(RESTORE_JSON_ABS_PATH).isFile()) + if (!fs.statSync(RESTORE_INSTRUCTIONS_ABS_PATH).isFile()) throw new Error('restore instructions is not a file!!') }) } diff --git a/src/user/system/CaptainManager.ts b/src/user/system/CaptainManager.ts index 7af346c..fb5a845 100644 --- a/src/user/system/CaptainManager.ts +++ b/src/user/system/CaptainManager.ts @@ -1,5 +1,4 @@ import uuid = require('uuid/v4') -import SshClientImport = require('ssh2') import request = require('request') import fs = require('fs-extra') import CaptainConstants = require('../../utils/CaptainConstants') @@ -18,7 +17,6 @@ import Authenticator = require('../Authenticator') import BackupManager from './BackupManager' const DEBUG_SALT = 'THIS IS NOT A REAL CERTIFICATE' -const SshClient = SshClientImport.Client const MAX_FAIL_ALLOWED = 4 const HEALTH_CHECK_INTERVAL = 20000 // ms @@ -649,102 +647,6 @@ class CaptainManager { }) } - joinDockerNode( - captainIpAddress: string, - isManager: boolean, - remoteNodeIpAddress: string, - privateKey: string - ) { - const remoteUserName = 'root' // Docker requires root access. It has to be root. - const dockerApi = this.dockerApi - - return Promise.resolve() - .then(function() { - return dockerApi.getJoinToken(isManager) - }) - .then(function(token) { - return new Promise(function(resolve, reject) { - const conn = new SshClient() - conn.on('error', function(err) { - Logger.e(err) - reject( - ApiStatusCodes.createError( - ApiStatusCodes.STATUS_ERROR_GENERIC, - 'SSH Connection error!!' - ) - ) - }) - .on('ready', function() { - Logger.d('SSH Client :: ready') - conn.exec( - CaptainConstants.disableFirewallCommand + - ' ' + - dockerApi.createJoinCommand( - captainIpAddress, - token - ), - function(err, stream) { - if (err) { - Logger.e(err) - reject( - ApiStatusCodes.createError( - ApiStatusCodes.STATUS_ERROR_GENERIC, - 'SSH Running command failed!!' - ) - ) - return - } - - let hasExisted = false - - stream - .on('close', function( - code: string, - signal: string - ) { - Logger.d( - 'Stream :: close :: code: ' + - code + - ', signal: ' + - signal - ) - conn.end() - if (hasExisted) { - return - } - hasExisted = true - resolve() - }) - .on('data', function(data: string) { - Logger.d('STDOUT: ' + data) - }) - .stderr.on('data', function(data) { - Logger.e('STDERR: ' + data) - if (hasExisted) { - return - } - hasExisted = true - reject( - ApiStatusCodes.createError( - ApiStatusCodes.STATUS_ERROR_GENERIC, - 'Error during setup: ' + - data - ) - ) - }) - } - ) - }) - .connect({ - host: remoteNodeIpAddress, - port: 22, - username: remoteUserName, - privateKey: privateKey, - }) - }) - }) - } - getLoadBalanceManager() { return this.loadBalancerManager } diff --git a/src/utils/CaptainInstaller.ts b/src/utils/CaptainInstaller.ts index 0753bb5..7b3f8e8 100644 --- a/src/utils/CaptainInstaller.ts +++ b/src/utils/CaptainInstaller.ts @@ -259,7 +259,7 @@ export function install() { }) .then(function(swarmId: string) { console.log('Swarm started: ' + swarmId) - return backupManger.startRestorationIfNeededPhase1() + return backupManger.startRestorationIfNeededPhase1(myIp4) }) .then(function() { return DockerApi.get().getLeaderNodeId() diff --git a/src/utils/Utils.ts b/src/utils/Utils.ts index db0580b..cb14870 100644 --- a/src/utils/Utils.ts +++ b/src/utils/Utils.ts @@ -9,6 +9,14 @@ export default class Utils { return req.method !== 'GET' } + static getDelayedPromise(time: number) { + return new Promise((res, rej) => { + setTimeout(() => { + res() + }, time) + }) + } + static getNeverReturningPromise() { return new Promise((res, rej) => { //