vitalif
/
f-stack
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix nginx coredump on virtio vm and support kni. 

1.close #67.
2.replace ff_primary process to primary worker process(the first one to
start and the last one to exit).
3.it's not graceful to reload.
dev
logwang 2017-09-05 20:24:51 +08:00
parent 1f5381c9ea
commit 63fb3d1fbf
3 changed files with 76 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
app/nginx-1.11.10/src/event/modules/ngx_ff_channel.c
View File

@ -46,18 +46,14 @@ static ngx_int_t ngx_ff_epoll_process_events(ngx_cycle_t *cycle,
ngx_msec_t timer, ngx_uint_t flags);
static ngx_int_t ngx_ff_create_connection(ngx_cycle_t *cycle);
static void ngx_ff_delete_connection();
static void ngx_ff_primary_channel_handler(ngx_event_t *ev);
static void ngx_ff_worker_channel_handler(ngx_event_t *ev);
static void *ngx_ff_channel_thread_main(void *args);
static ngx_int_t ngx_ff_add_channel_event(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event, ngx_event_handler_pt handler);
static ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event);
ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event);
ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
struct channel_thread_args {
ngx_cycle_t *cycle;
@ -650,79 +646,6 @@ ngx_ff_worker_channel_handler(ngx_event_t *ev)
}
}
static void
ngx_ff_primary_channel_handler(ngx_event_t *ev)
{
ngx_int_t n;
ngx_channel_t ch;
ngx_connection_t *c;
if (ev->timedout) {
ev->timedout = 0;
return;
}
c = ev->data;
ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "primary channel handler");
for ( ;; ) {
n = ngx_read_channel(c->fd, &ch, sizeof(ngx_channel_t), ev->log);
ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, "channel: %i", n);
if (n == NGX_ERROR) {
ngx_ff_epoll_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
close(c->fd);
ngx_ff_delete_connection();
return;
}
if (n == NGX_AGAIN) {
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
"channel command: %ui", ch.command);
switch (ch.command) {
case NGX_CMD_TERMINATE:
ngx_terminate = 1;
break;
case NGX_CMD_REOPEN:
ngx_reopen = 1;
break;
case NGX_CMD_OPEN_CHANNEL:
ngx_log_debug3(NGX_LOG_DEBUG_CORE, ev->log, 0,
"get channel s:%i pid:%P fd:%d",
ch.slot, ch.pid, ch.fd);
ngx_processes[ch.slot].pid = ch.pid;
ngx_processes[ch.slot].channel[0] = ch.fd;
break;
case NGX_CMD_CLOSE_CHANNEL:
ngx_log_debug4(NGX_LOG_DEBUG_CORE, ev->log, 0,
"close channel s:%i pid:%P our:%P fd:%d",
ch.slot, ch.pid, ngx_processes[ch.slot].pid,
ngx_processes[ch.slot].channel[0]);
if (close(ngx_processes[ch.slot].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
"close() channel failed");
}
ngx_processes[ch.slot].channel[0] = -1;
break;
}
}
}
static void *
ngx_ff_channel_thread_main(void *args)
{
@ -746,7 +669,7 @@ ngx_ff_channel_thread_main(void *args)
return NULL;
}
ngx_int_t
static ngx_int_t
ngx_ff_process_channel_events(ngx_cycle_t *cycle)
{
return ngx_ff_epoll_process_events(cycle, 500, NGX_UPDATE_TIME);
@ -769,7 +692,7 @@ ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd,
cta->event = event;
cta->handler = ngx_ff_worker_channel_handler;
ret = pthread_create(&channel_thread, NULL,
ret = pthread_create(&channel_thread, NULL,
ngx_ff_channel_thread_main, (void *)cta);
if (ret != 0) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
@ -782,12 +705,4 @@ ngx_ff_start_worker_channel(ngx_cycle_t *cycle, ngx_fd_t fd,
return NGX_OK;
}
ngx_int_t
ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event)
{
return ngx_ff_add_channel_event(cycle, fd, event,
ngx_ff_primary_channel_handler);
}
#endif

189
app/nginx-1.11.10/src/os/unix/ngx_process_cycle.c
View File

@ -25,9 +25,6 @@ static void ngx_worker_process_exit(ngx_cycle_t *cycle);
#if (NGX_HAVE_FSTACK)
extern ngx_int_t ngx_ff_start_worker_channel(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event);
extern ngx_int_t ngx_ff_start_primary_channel(ngx_cycle_t *cycle,
ngx_fd_t fd, ngx_int_t event);
extern ngx_int_t ngx_ff_process_channel_events(ngx_cycle_t *cycle);
#else
static void ngx_channel_handler(ngx_event_t *ev);
#endif
@ -80,141 +77,7 @@ static ngx_log_t ngx_exit_log;
static ngx_open_file_t ngx_exit_log_file;
#if (NGX_HAVE_FSTACK)
static void
ngx_ff_primary_process_exit(ngx_cycle_t *cycle)
{
ngx_uint_t i;
for (i = 0; cycle->modules[i]; i++) {
if (cycle->modules[i]->exit_process) {
cycle->modules[i]->exit_process(cycle);
}
}
/*
* Copy ngx_cycle->log related data to the special static exit cycle,
* log, and log file structures enough to allow a signal handler to log.
* The handler may be called when standard ngx_cycle->log allocated from
* ngx_cycle->pool is already destroyed.
*/
ngx_exit_log = *ngx_log_get_file_log(ngx_cycle->log);
ngx_exit_log_file.fd = ngx_exit_log.file->fd;
ngx_exit_log.file = &ngx_exit_log_file;
ngx_exit_log.next = NULL;
ngx_exit_log.writer = NULL;
ngx_exit_cycle.log = &ngx_exit_log;
ngx_exit_cycle.files = ngx_cycle->files;
ngx_exit_cycle.files_n = ngx_cycle->files_n;
ngx_cycle = &ngx_exit_cycle;
ngx_destroy_pool(cycle->pool);
// wait worker process exited.
ngx_msleep(500);
ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit");
exit(0);
}
static void
ngx_ff_primary_process_cycle(ngx_cycle_t *cycle, void *data)
{
ngx_core_conf_t *ccf;
ngx_uint_t i;
ngx_int_t n;
ngx_process = NGX_PROCESS_WORKER;
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,
ngx_core_module);
ngx_setproctitle("ff primary process");
if (ff_mod_init((const char *)ccf->fstack_conf.data, 0, 1)) {
exit(2);
}
for (i = 0; cycle->modules[i]; i++) {
if (cycle->modules[i]->init_process) {
if (cycle->modules[i]->init_process(cycle) == NGX_ERROR) {
/* fatal */
exit(2);
}
}
}
for (n = 0; n < ngx_last_process; n++) {
if (ngx_processes[n].pid == -1) {
continue;
}
if (n == ngx_process_slot) {
continue;
}
if (ngx_processes[n].channel[1] == -1) {
continue;
}
if (close(ngx_processes[n].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"close() channel failed");
}
}
if (close(ngx_processes[ngx_process_slot].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"close() channel failed");
}
if (ngx_ff_start_primary_channel(cycle, ngx_channel, NGX_READ_EVENT)) {
exit(2);
}
for ( ;; ) {
if (ngx_terminate) {
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting");
ngx_ff_primary_process_exit(cycle);
}
if (ngx_reopen) {
ngx_reopen = 0;
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");
ngx_reopen_files(cycle, -1);
}
(void) ngx_ff_process_channel_events(cycle);
}
}
static void
ngx_start_ff_primary_process(ngx_cycle_t *cycle)
{
ngx_channel_t ch;
ngx_memzero(&ch, sizeof(ngx_channel_t));
ch.command = NGX_CMD_OPEN_CHANNEL;
ngx_spawn_process(cycle, ngx_ff_primary_process_cycle,
NULL, "ff primary process",
NGX_PROCESS_RESPAWN);
ch.pid = ngx_processes[ngx_process_slot].pid;
ch.slot = ngx_process_slot;
ch.fd = ngx_processes[ngx_process_slot].channel[0];
ngx_pass_open_channel(cycle, &ch);
// wait for primary process startup.
ngx_sleep(1);
}
static ngx_int_t ngx_ff_primary;
#endif
void
@ -226,6 +89,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
ngx_int_t i;
#if (NGX_HAVE_FSTACK)
ngx_uint_t sigio;
ngx_uint_t sig_worker_quit = 0;
#else
ngx_uint_t n, sigio;
#endif
@ -281,16 +145,6 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
#if (NGX_HAVE_FSTACK)
if (ccf->fstack_conf.len == 0) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"fstack_conf null");
exit(2);
}
ngx_start_ff_primary_process(cycle);
#endif
ngx_start_worker_processes(cycle, ccf->worker_processes,
NGX_PROCESS_RESPAWN);
ngx_start_cache_manager_processes(cycle, 0);
@ -383,6 +237,20 @@ ngx_master_process_cycle(ngx_cycle_t *cycle)
}
if (ngx_reconfigure) {
#if (NGX_HAVE_FSTACK)
if (!sig_worker_quit) {
sig_worker_quit = 1;
ngx_signal_worker_processes(cycle,
ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
continue;
}
if (live) {
continue;
}
sig_worker_quit = 0;
#endif
ngx_reconfigure = 0;
if (ngx_new_binary) {
@ -527,6 +395,12 @@ ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, ngx_int_t type)
ch.fd = ngx_processes[ngx_process_slot].channel[0];
ngx_pass_open_channel(cycle, &ch);
#if (NGX_HAVE_FSTACK)
if (i == 0) {
// wait for ff_primary worker process startup.
ngx_sleep(1);
}
#endif
}
}
@ -780,7 +654,11 @@ ngx_reap_children(ngx_cycle_t *cycle)
if (ngx_processes[i].respawn
&& !ngx_processes[i].exiting
&& !ngx_terminate
&& !ngx_quit)
&& !ngx_quit
#if (NGX_HAVE_FSTACK)
&& !ngx_reconfigure
#endif
)
{
if (ngx_spawn_process(cycle, ngx_processes[i].proc,
ngx_processes[i].data,
@ -1115,7 +993,12 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker)
exit(2);
}
if (ff_mod_init((const char *)ccf->fstack_conf.data, worker, 0)) {
if (worker == 0) {
ngx_ff_primary = 1;
}
if (ff_mod_init((const char *)ccf->fstack_conf.data, worker,
ngx_ff_primary)) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"ff_mod_init failed");
exit(2);
@ -1247,6 +1130,12 @@ ngx_worker_process_exit(ngx_cycle_t *cycle)
ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "exit");
#if (NGX_HAVE_FSTACK)
if (ngx_ff_primary) {
// wait for secondary worker processes to exit.
ngx_msleep(500);
}
#endif
exit(0);
}

73
doc/F-Stack_Nginx_APP_Guide.md
View File

@ -8,52 +8,47 @@ F-Stack is an open source network framework based on DPDK. F-Stack supports stan
```
+--------+
|
+------------------------+ |
channel: socketpair | signal(reload, quit..)
+------------------------+ |
|
+---------v--------+
| |
+----------------+ master process +---------------+
| | | |
| channel +----------+-------+ |
| | channel |
| channel | |
| | |
+---------+----------+ +----------+--------+ +---------+--------+
| | | | | |
| ff primary process | | worker process | | worker process |
| | | | | |
+--------------------+ +-------------------+ +------------------+
ff_init(primary) +--------+ +-------+
loop: | | | |
handle channel event | fstack | |channel|
| main | | event |
| loop | |thread |
| thread | | |
| | | |
+--------+ +-------+
woker loop:
process handle
cycle channel
event
+--------+
+------------------------+ |
channel: socketpair |
+------------------------+ | signal(reload, quit..)
|
|
+---------v--------+
| |
+----------------+ master process +---------------+
| | | |
| channel +----------+-------+ |
| | channel |
| channel | |
| | |
+---------+----------+ +----------+--------+ +---------+--------+
first one to start | | | | | |
last one to exit<-+ primary worker | | secondary worker | | secondary worker |
| | | | | |
+--------------------+ +-------------------+ +------------------+
+--------+ +-------+ +--------+ +-------+
| | | | | | | |
| fstack | |channel| | fstack | |channel|
| main | | event | | main | | event |
| loop | |thread | | loop | |thread |
| thread | | | | thread | | |
| | | | | | | |
+--------+ +-------+ +--------+ +-------+
woker loop: worker loop:
process handle process handle
cycle channel cycle channel
event event
```
- spawn an extra process ff primary: ff_init(primary);loop(handle channel event).This process doesn't handle SIGQUIT.
- spawn primary worker firstly, and then wait for primary startup, continue to spawn secondary workers.
- worker process has 2 threads. main thread: ff_init(secondary);ff_run(worker_process_cycle), channel thread: loop(handle channel event).
- worker process has 2 threads. main thread: ff_init();ff_run(worker_process_cycle), channel thread: loop(handle channel event).
Note that:
- kni couldn't be enabled in this version, because kni is only processed by primary and worker `ff_primary` won't execute `ff_run()`.
- supported nginx signals: reload(HUP)/reopen(USR1)/stop(TERM).
- unsupported nginx signals: NGX_CHANGEBIN_SIGNAL(USR2).
- when use `nginx -s reload`, you should make sure that `woker_processes` in nginx.conf and f-stack.conf haven't be modified.
- the `reload` is not graceful, service will still be unavailable during the process of reloading.
- necessary modifies in nginx.conf: