vitalif
/
vitastor
5
5
Fork 7
Code Issues 6 Pull Requests 2 Releases 35 Wiki Activity
Browse Source

Use the same etcd address selection algorithm in the monitor

test-assert
Vitaliy Filippov 7 months ago
parent
ce5b6253ab
commit
a8f5c71ae8
2 changed files with 90 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      mon/mon-main.js
  2. 107
      mon/mon.js

17
mon/mon-main.js
View File

@ -9,17 +9,18 @@ const options = {};
for (let i = 2; i < process.argv.length; i++)
{
if (process.argv[i].substr(0, 2) == '--')
if (process.argv[i] === '-h' || process.argv[i] === '--help')
{
console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+
' [--etcd_address "http://127.0.0.1:2379,..."] [--config_file /etc/vitastor/vitastor.conf]'+
' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5]');
process.exit();
}
else if (process.argv[i].substr(0, 2) == '--')
{
options[process.argv[i].substr(2)] = process.argv[i+1];
i++;
}
}
if (!options.etcd_url)
{
console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' --etcd_url "http://127.0.0.1:2379,..." --etcd_prefix "/vitastor" --etcd_start_timeout 5 [--verbose 1]');
process.exit();
}
new Mon(options).start().catch(e => { console.error(e); process.exit(); });
new Mon(options).start().catch(e => { console.error(e); process.exit(1); });

107
mon/mon.js
View File

@ -1,6 +1,7 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
const fs = require('fs');
const http = require('http');
const crypto = require('crypto');
const os = require('os');
@ -323,17 +324,17 @@ class Mon
{
constructor(config)
{
// FIXME: Maybe prefer local etcd
this.etcd_urls = [];
for (let url of config.etcd_url.split(/,/))
this.die = (e) => this._die(e);
if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
{
let scheme = 'http';
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
if (!/\/[^\/]/.exec(url))
url += '/v3';
this.etcd_urls.push(scheme+'://'+url);
config = {
...JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' })),
...config,
};
}
this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
this.verbose = config.verbose || 0;
this.initConfig = config;
this.config = {};
this.etcd_prefix = config.etcd_prefix || '/vitastor';
this.etcd_prefix = this.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1');
@ -343,6 +344,35 @@ class Mon
this.on_stop_cb = () => this.on_stop().catch(console.error);
}
parse_etcd_addresses(addrs)
{
const is_local_ip = this.local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
this.etcd_local = [];
this.etcd_urls = [];
this.selected_etcd_url = null;
this.etcd_urls_to_try = [];
if (!(addrs instanceof Array))
addrs = addrs ? (''+(addrs||'')).split(/,/) : [];
if (!addrs.length)
{
console.error('Vitastor etcd address(es) not specified. Please set on the command line or in the config file');
process.exit(1);
}
for (let url of addrs)
{
let scheme = 'http';
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
const slash = url.indexOf('/');
const colon = url.indexOf(':');
const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
url = scheme+'://'+(slash >= 0 ? url : url+'/v3');
if (is_local)
this.etcd_local.push(url);
else
this.etcd_urls.push(url);
}
}
async start()
{
await this.load_config();
@ -411,6 +441,31 @@ class Mon
}
}
pick_next_etcd()
{
if (this.selected_etcd_url)
return this.selected_etcd_url;
if (!this.etcd_urls_to_try || !this.etcd_urls_to_try.length)
{
this.etcd_urls_to_try = [ ...this.etcd_local ];
const others = [ ...this.etcd_urls ];
while (others.length)
{
const url = others.splice(0|(others.length*Math.random()), 1);
this.etcd_urls_to_try.push(url[0]);
}
}
this.selected_etcd_url = this.etcd_urls_to_try.shift();
return this.selected_etcd_url;
}
restart_watcher(cur_addr)
{
if (this.selected_etcd_url == cur_addr)
this.selected_etcd_url = null;
this.start_watcher(this.config.etcd_mon_retries).catch(this.die);
}
async start_watcher(retries)
{
let retry = 0;
@ -420,7 +475,8 @@ class Mon
}
while (retries < 0 || retry < retries)
{
const base = 'ws'+this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)].substr(4);
const cur_addr = this.pick_next_etcd();
const base = 'ws'+cur_addr.substr(4);
const ok = await new Promise((ok, no) =>
{
const timer_id = setTimeout(() =>
@ -443,9 +499,9 @@ class Mon
});
});
if (ok)
{
break;
}
if (this.selected_etcd_url == cur_addr)
this.selected_etcd_url = null;
this.ws = null;
retry++;
}
@ -453,6 +509,8 @@ class Mon
{
this.die('Failed to open etcd watch websocket');
}
const cur_addr = this.selected_etcd_url;
this.ws.on('error', () => this.restart_watcher(cur_addr));
this.ws.send(JSON.stringify({
create_request: {
key: b64(this.etcd_prefix+'/'),
@ -510,7 +568,7 @@ class Mon
}
if (pg_states_changed)
{
this.save_last_clean().catch(console.error);
this.save_last_clean().catch(this.die);
}
if (stats_changed)
{
@ -1196,7 +1254,7 @@ class Mon
this.recheck_timer = setTimeout(() =>
{
this.recheck_timer = null;
this.recheck_pgs().catch(console.error);
this.recheck_pgs().catch(this.die);
}, this.config.mon_change_timeout || 1000);
}
@ -1463,7 +1521,7 @@ class Mon
cur[key_parts[key_parts.length-1]] = kv.value;
if (key === 'config/global')
{
this.config = this.state.config.global;
this.config = { ...this.initConfig, ...this.state.config.global };
this.check_config();
for (const osd_num in this.state.osd.stats)
{
@ -1500,12 +1558,15 @@ class Mon
}
while (retries < 0 || retry < retries)
{
const base = this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)];
retry++;
const base = this.pick_next_etcd();
const res = await POST(base+path, body, timeout);
if (res.error)
{
console.error('etcd returned error: '+res.error);
break;
if (this.selected_etcd_url == base)
this.selected_etcd_url = null;
console.error('failed to query etcd: '+res.error);
continue;
}
if (res.json)
{
@ -1514,26 +1575,20 @@ class Mon
console.error('etcd returned error: '+res.json.error);
break;
}
if (this.etcd_urls.length > 1)
{
// Stick to the same etcd for the rest of calls
this.etcd_urls = [ base ];
}
return res.json;
}
retry++;
}
this.die();
}
die(err)
_die(err)
{
// In fact we can just try to rejoin
console.error(new Error(err || 'Cluster connection failed'));
process.exit(1);
}
local_ips()
local_ips(all)
{
const ips = [];
const ifaces = os.networkInterfaces();
@ -1541,7 +1596,7 @@ class Mon
{
for (const iface of ifaces[ifname])
{
if (iface.family == 'IPv4' && !iface.internal)
if (iface.family == 'IPv4' && !iface.internal || all)
{
ips.push(iface.address);
}

Write Preview
Loading…
Cancel
Save