From a8f5c71ae8463b3a64ab7884675e23b555dbb06e Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 28 Nov 2021 01:19:42 +0300 Subject: [PATCH] Use the same etcd address selection algorithm in the monitor --- mon/mon-main.js | 17 ++++---- mon/mon.js | 107 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 90 insertions(+), 34 deletions(-) diff --git a/mon/mon-main.js b/mon/mon-main.js index 5e83b181..571a0269 100755 --- a/mon/mon-main.js +++ b/mon/mon-main.js @@ -9,17 +9,18 @@ const options = {}; for (let i = 2; i < process.argv.length; i++) { - if (process.argv[i].substr(0, 2) == '--') + if (process.argv[i] === '-h' || process.argv[i] === '--help') + { + console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+ + ' [--etcd_address "http://127.0.0.1:2379,..."] [--config_file /etc/vitastor/vitastor.conf]'+ + ' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5]'); + process.exit(); + } + else if (process.argv[i].substr(0, 2) == '--') { options[process.argv[i].substr(2)] = process.argv[i+1]; i++; } } -if (!options.etcd_url) -{ - console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' --etcd_url "http://127.0.0.1:2379,..." --etcd_prefix "/vitastor" --etcd_start_timeout 5 [--verbose 1]'); - process.exit(); -} - -new Mon(options).start().catch(e => { console.error(e); process.exit(); }); +new Mon(options).start().catch(e => { console.error(e); process.exit(1); }); diff --git a/mon/mon.js b/mon/mon.js index ed3baded..cbc9c7da 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -1,6 +1,7 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.1 (see README.md for details) +const fs = require('fs'); const http = require('http'); const crypto = require('crypto'); const os = require('os'); @@ -323,17 +324,17 @@ class Mon { constructor(config) { - // FIXME: Maybe prefer local etcd - this.etcd_urls = []; - for (let url of config.etcd_url.split(/,/)) + this.die = (e) => this._die(e); + if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf')) { - let scheme = 'http'; - url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; }); - if (!/\/[^\/]/.exec(url)) - url += '/v3'; - this.etcd_urls.push(scheme+'://'+url); + config = { + ...JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' })), + ...config, + }; } + this.parse_etcd_addresses(config.etcd_address||config.etcd_url); this.verbose = config.verbose || 0; + this.initConfig = config; this.config = {}; this.etcd_prefix = config.etcd_prefix || '/vitastor'; this.etcd_prefix = this.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1'); @@ -343,6 +344,35 @@ class Mon this.on_stop_cb = () => this.on_stop().catch(console.error); } + parse_etcd_addresses(addrs) + { + const is_local_ip = this.local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {}); + this.etcd_local = []; + this.etcd_urls = []; + this.selected_etcd_url = null; + this.etcd_urls_to_try = []; + if (!(addrs instanceof Array)) + addrs = addrs ? (''+(addrs||'')).split(/,/) : []; + if (!addrs.length) + { + console.error('Vitastor etcd address(es) not specified. Please set on the command line or in the config file'); + process.exit(1); + } + for (let url of addrs) + { + let scheme = 'http'; + url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; }); + const slash = url.indexOf('/'); + const colon = url.indexOf(':'); + const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)]; + url = scheme+'://'+(slash >= 0 ? url : url+'/v3'); + if (is_local) + this.etcd_local.push(url); + else + this.etcd_urls.push(url); + } + } + async start() { await this.load_config(); @@ -411,6 +441,31 @@ class Mon } } + pick_next_etcd() + { + if (this.selected_etcd_url) + return this.selected_etcd_url; + if (!this.etcd_urls_to_try || !this.etcd_urls_to_try.length) + { + this.etcd_urls_to_try = [ ...this.etcd_local ]; + const others = [ ...this.etcd_urls ]; + while (others.length) + { + const url = others.splice(0|(others.length*Math.random()), 1); + this.etcd_urls_to_try.push(url[0]); + } + } + this.selected_etcd_url = this.etcd_urls_to_try.shift(); + return this.selected_etcd_url; + } + + restart_watcher(cur_addr) + { + if (this.selected_etcd_url == cur_addr) + this.selected_etcd_url = null; + this.start_watcher(this.config.etcd_mon_retries).catch(this.die); + } + async start_watcher(retries) { let retry = 0; @@ -420,7 +475,8 @@ class Mon } while (retries < 0 || retry < retries) { - const base = 'ws'+this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)].substr(4); + const cur_addr = this.pick_next_etcd(); + const base = 'ws'+cur_addr.substr(4); const ok = await new Promise((ok, no) => { const timer_id = setTimeout(() => @@ -443,9 +499,9 @@ class Mon }); }); if (ok) - { break; - } + if (this.selected_etcd_url == cur_addr) + this.selected_etcd_url = null; this.ws = null; retry++; } @@ -453,6 +509,8 @@ class Mon { this.die('Failed to open etcd watch websocket'); } + const cur_addr = this.selected_etcd_url; + this.ws.on('error', () => this.restart_watcher(cur_addr)); this.ws.send(JSON.stringify({ create_request: { key: b64(this.etcd_prefix+'/'), @@ -510,7 +568,7 @@ class Mon } if (pg_states_changed) { - this.save_last_clean().catch(console.error); + this.save_last_clean().catch(this.die); } if (stats_changed) { @@ -1196,7 +1254,7 @@ class Mon this.recheck_timer = setTimeout(() => { this.recheck_timer = null; - this.recheck_pgs().catch(console.error); + this.recheck_pgs().catch(this.die); }, this.config.mon_change_timeout || 1000); } @@ -1463,7 +1521,7 @@ class Mon cur[key_parts[key_parts.length-1]] = kv.value; if (key === 'config/global') { - this.config = this.state.config.global; + this.config = { ...this.initConfig, ...this.state.config.global }; this.check_config(); for (const osd_num in this.state.osd.stats) { @@ -1500,12 +1558,15 @@ class Mon } while (retries < 0 || retry < retries) { - const base = this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)]; + retry++; + const base = this.pick_next_etcd(); const res = await POST(base+path, body, timeout); if (res.error) { - console.error('etcd returned error: '+res.error); - break; + if (this.selected_etcd_url == base) + this.selected_etcd_url = null; + console.error('failed to query etcd: '+res.error); + continue; } if (res.json) { @@ -1514,26 +1575,20 @@ class Mon console.error('etcd returned error: '+res.json.error); break; } - if (this.etcd_urls.length > 1) - { - // Stick to the same etcd for the rest of calls - this.etcd_urls = [ base ]; - } return res.json; } - retry++; } this.die(); } - die(err) + _die(err) { // In fact we can just try to rejoin console.error(new Error(err || 'Cluster connection failed')); process.exit(1); } - local_ips() + local_ips(all) { const ips = []; const ifaces = os.networkInterfaces(); @@ -1541,7 +1596,7 @@ class Mon { for (const iface of ifaces[ifname]) { - if (iface.family == 'IPv4' && !iface.internal) + if (iface.family == 'IPv4' && !iface.internal || all) { ips.push(iface.address); }