[lttng-tools,v2,4/6] Allow regenerating the statedump of a running session

The "lttng regenerate statedump" command can be used to regenerate the
statedump of a running session whenever needed. This is particularly
useful in snapshot and trace-file rotation modes where the original
statedump may be lost.

Signed-off-by: Julien Desfossez <jdesfossez at efficios.com>
---
 include/lttng/lttng-error.h              |  1 +
 include/lttng/lttng.h                    |  9 +++++
 src/bin/lttng-sessiond/cmd.c             | 44 +++++++++++++++++++++
 src/bin/lttng-sessiond/cmd.h             |  1 +
 src/bin/lttng-sessiond/main.c            |  6 +++
 src/bin/lttng-sessiond/ust-app.c         | 66 ++++++++++++++++++++++++++++++++
 src/bin/lttng-sessiond/ust-app.h         |  6 +++
 src/bin/lttng/commands/regenerate.c      | 19 +++++++++
 src/common/error.c                       |  1 +
 src/common/kernel-ctl/kernel-ctl.c       |  5 +++
 src/common/kernel-ctl/kernel-ctl.h       |  1 +
 src/common/sessiond-comm/sessiond-comm.h |  1 +
 src/lib/lttng-ctl/lttng-ctl.c            | 30 +++++++++++++++
 13 files changed, 190 insertions(+)

On Mon, Jul 4, 2016 at 11:13 AM, Julien Desfossez
<jdesfossez at efficios.com> wrote:
> The "lttng regenerate statedump" command can be used to regenerate the
> statedump of a running session whenever needed. This is particularly
> useful in snapshot and trace-file rotation modes where the original
> statedump may be lost.
>
> Signed-off-by: Julien Desfossez <jdesfossez at efficios.com>
> ---
>  include/lttng/lttng-error.h              |  1 +
>  include/lttng/lttng.h                    |  9 +++++
>  src/bin/lttng-sessiond/cmd.c             | 44 +++++++++++++++++++++
>  src/bin/lttng-sessiond/cmd.h             |  1 +
>  src/bin/lttng-sessiond/main.c            |  6 +++
>  src/bin/lttng-sessiond/ust-app.c         | 66 ++++++++++++++++++++++++++++++++
>  src/bin/lttng-sessiond/ust-app.h         |  6 +++
>  src/bin/lttng/commands/regenerate.c      | 19 +++++++++
>  src/common/error.c                       |  1 +
>  src/common/kernel-ctl/kernel-ctl.c       |  5 +++
>  src/common/kernel-ctl/kernel-ctl.h       |  1 +
>  src/common/sessiond-comm/sessiond-comm.h |  1 +
>  src/lib/lttng-ctl/lttng-ctl.c            | 30 +++++++++++++++
>  13 files changed, 190 insertions(+)
>
> diff --git a/include/lttng/lttng-error.h b/include/lttng/lttng-error.h
> index 72194ef..52ce333 100644
> --- a/include/lttng/lttng-error.h
> +++ b/include/lttng/lttng-error.h
> @@ -142,6 +142,7 @@ enum lttng_error_code {
>         LTTNG_ERR_LIVE_SESSION           = 119, /* Live session unsupported */
>         LTTNG_ERR_PER_PID_SESSION        = 120, /* Per-PID sessions unsupported */
>         LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE = 121, /* Context unavailable on this kernel */
> +       LTTNG_ERR_REGEN_STATEDUMP_FAIL   = 122, /* Failed to regenerate the statdump */

statdump -> statedump

>
>         /* MUST be last element */
>         LTTNG_ERR_NR,                           /* Last element */
> diff --git a/include/lttng/lttng.h b/include/lttng/lttng.h
> index 09aa969..b81a01c 100644
> --- a/include/lttng/lttng.h
> +++ b/include/lttng/lttng.h
> @@ -173,6 +173,15 @@ extern int lttng_metadata_regenerate(const char *session_name);
>   */
>  extern int lttng_regenerate_metadata(const char *session_name);
>
> +/*
> + * Trigger the regeneration of the statedump for a session. The new statedump
> + * information is appended to the currently active trace, the session needs to
> + * be started.

needs to be active

> + *
> + * Return 0 on success, a negative LTTng error code on error.
> + */
> +extern int lttng_regenerate_statedump(const char *session_name);
> +
>  #ifdef __cplusplus
>  }
>  #endif
> diff --git a/src/bin/lttng-sessiond/cmd.c b/src/bin/lttng-sessiond/cmd.c
> index bd63389..480333d 100644
> --- a/src/bin/lttng-sessiond/cmd.c
> +++ b/src/bin/lttng-sessiond/cmd.c
> @@ -3540,6 +3540,50 @@ end:
>  }
>
>  /*
> + * Command LTTNG_REGENERATE_STATEDUMP from the lttng-ctl library.
> + *
> + * Ask the tracer to regenerate a new statedump.
> + *
> + * Return 0 on success or else a LTTNG_ERR code.
> + */
> +int cmd_regenerate_statedump(struct ltt_session *session)
> +{
> +       int ret;
> +
> +       assert(session);
> +
> +       if (!session->active) {
> +               ret = LTTNG_ERR_SESSION_NOT_STARTED;
> +               goto end;
> +       }
> +       ret = 0;
> +
> +       if (session->kernel_session) {
> +               ret = kernctl_session_regenerate_statedump(
> +                               session->kernel_session->fd);
> +               if (ret < 0) {
> +                       ret = LTTNG_ERR_REGEN_STATEDUMP_FAIL;

Does kernctl_session_regenerate_statedump() return a more precise error code?
I see that this can fail if the session is not active (handled
before). Are there other reasons?

> +                       ERR("Failed to regenerate the kernel statedump");
> +                       goto end;
> +               }
> +       }
> +
> +       if (session->ust_session) {
> +               ret = ust_app_regenerate_statedump_all(session->ust_session);

Same applies here.

> +               if (ret < 0) {
> +                       ret = LTTNG_ERR_REGEN_STATEDUMP_FAIL;
> +                       ERR("Failed to regenerate the UST statedump");
> +                       goto end;
> +               }
> +       }
> +       DBG("Cmd regenerate statedump for session %s", session->name);
> +       ret = LTTNG_OK;
> +
> +end:
> +       return ret;
> +}
> +
> +/*
>   * Send relayd sockets from snapshot output to consumer. Ignore request if the
>   * snapshot output is *not* set with a remote destination.
>   *
> diff --git a/src/bin/lttng-sessiond/cmd.h b/src/bin/lttng-sessiond/cmd.h
> index 320d717..975a7f1 100644
> --- a/src/bin/lttng-sessiond/cmd.h
> +++ b/src/bin/lttng-sessiond/cmd.h
> @@ -111,5 +111,6 @@ int cmd_snapshot_record(struct ltt_session *session,
>  int cmd_set_session_shm_path(struct ltt_session *session,
>                 const char *shm_path);
>  int cmd_regenerate_metadata(struct ltt_session *session);
> +int cmd_regenerate_statedump(struct ltt_session *session);
>
>  #endif /* CMD_H */
> diff --git a/src/bin/lttng-sessiond/main.c b/src/bin/lttng-sessiond/main.c
> index 3623e5d..8309c3f 100644
> --- a/src/bin/lttng-sessiond/main.c
> +++ b/src/bin/lttng-sessiond/main.c
> @@ -3004,6 +3004,7 @@ static int process_client_msg(struct command_ctx *cmd_ctx, int sock,
>         case LTTNG_SAVE_SESSION:
>         case LTTNG_SET_SESSION_SHM_PATH:
>         case LTTNG_REGENERATE_METADATA:
> +       case LTTNG_REGENERATE_STATEDUMP:
>                 need_domain = 0;
>                 break;
>         default:
> @@ -4119,6 +4120,11 @@ error_add_context:
>                 ret = cmd_regenerate_metadata(cmd_ctx->session);
>                 break;
>         }
> +       case LTTNG_REGENERATE_STATEDUMP:
> +       {
> +               ret = cmd_regenerate_statedump(cmd_ctx->session);
> +               break;
> +       }
>         default:
>                 ret = LTTNG_ERR_UND;
>                 break;
> diff --git a/src/bin/lttng-sessiond/ust-app.c b/src/bin/lttng-sessiond/ust-app.c
> index f30df20..3474082 100644
> --- a/src/bin/lttng-sessiond/ust-app.c
> +++ b/src/bin/lttng-sessiond/ust-app.c
> @@ -6148,3 +6148,69 @@ end:
>         rcu_read_unlock();
>         return ret;
>  }
> +
> +static
> +int ust_app_regenerate_statedump(struct ltt_ust_session *usess,
> +               struct ust_app *app)
> +{
> +       int ret = 0;
> +       struct ust_app_session *ua_sess;
> +
> +       DBG("Regenerating the metadata for ust app pid %d", app->pid);
> +
> +       rcu_read_lock();
> +
> +       ua_sess = lookup_session_by_app(usess, app);
> +       if (ua_sess == NULL) {
> +               /* The session is in teardown process. Ignore and continue. */
> +               goto end;
> +       }
> +
> +       pthread_mutex_lock(&ua_sess->lock);
> +
> +       if (ua_sess->deleted) {
> +               pthread_mutex_unlock(&ua_sess->lock);

Remove this pthread_mutex_unlock() and goto end_unlock;

> +               goto end;
> +       }
> +
> +       pthread_mutex_lock(&app->sock_lock);
> +       ret = ustctl_regenerate_statedump(app->sock, ua_sess->handle);
> +       pthread_mutex_unlock(&app->sock_lock);

Add
end_unlock:

> +       pthread_mutex_unlock(&ua_sess->lock);
> +
> +end:
> +       rcu_read_unlock();
> +       health_code_update();
> +       return ret;
> +}
> +
> +/*
> + * Regenerate the statedump for each app in the session.
> + */
> +int ust_app_regenerate_statedump_all(struct ltt_ust_session *usess)
> +{
> +       int ret = 0;
> +       struct lttng_ht_iter iter;
> +       struct ust_app *app;
> +
> +       DBG("Regenerating the metadata for all UST apps");
> +
> +       rcu_read_lock();
> +
> +       cds_lfht_for_each_entry(ust_app_ht->ht, &iter.iter, app, pid_n.node) {
> +               if (!app->compatible) {
> +                       continue;
> +               }
> +
> +               ret = ust_app_regenerate_statedump(usess, app);
> +               if (ret < 0) {
> +                       /* Continue to next apps even on error */

to next app

> +                       continue;
> +               }
> +       }
> +
> +       rcu_read_unlock();
> +
> +       return 0;
> +}
> +

Remove blank line here.

> diff --git a/src/bin/lttng-sessiond/ust-app.h b/src/bin/lttng-sessiond/ust-app.h
> index d02f353..8369d14 100644
> --- a/src/bin/lttng-sessiond/ust-app.h
> +++ b/src/bin/lttng-sessiond/ust-app.h
> @@ -352,6 +352,7 @@ int ust_app_pid_get_channel_runtime_stats(struct ltt_ust_session *usess,
>                 struct ltt_ust_channel *uchan,
>                 struct consumer_output *consumer,
>                 int overwrite, uint64_t *discarded, uint64_t *lost);
> +int ust_app_regenerate_statedump_all(struct ltt_ust_session *usess);
>
>  static inline
>  int ust_app_supported(void)
> @@ -583,6 +584,11 @@ int ust_app_pid_get_channel_runtime_stats(struct ltt_ust_session *usess,
>  {
>         return 0;
>  }
> +static inline
> +int ust_app_regenerate_metadata_all(struct ltt_ust_session *usess)
> +{
> +       return 0;
> +}
>
>  #endif /* HAVE_LIBLTTNG_UST_CTL */
>
> diff --git a/src/bin/lttng/commands/regenerate.c b/src/bin/lttng/commands/regenerate.c
> index c64d8fe..f111d06 100644
> --- a/src/bin/lttng/commands/regenerate.c
> +++ b/src/bin/lttng/commands/regenerate.c
> @@ -32,6 +32,7 @@ static char *opt_session_name;
>  static char *session_name = NULL;
>
>  static int regenerate_metadata(int argc, const char **argv);
> +static int regenerate_statedump(int argc, const char **argv);
>
>  enum {
>         OPT_HELP = 1,
> @@ -52,6 +53,7 @@ static struct poptOption long_options[] = {
>
>  static struct cmd_struct actions[] = {
>         { "metadata", regenerate_metadata },
> +       { "statedump", regenerate_statedump },
>         { NULL, NULL }  /* Array closure */
>  };
>
> @@ -88,6 +90,23 @@ end:
>         return ret;
>  }
>
> +static int regenerate_statedump(int argc, const char **argv)
> +{
> +       int ret;
> +
> +       if (argc > 1) {
> +               ret = -LTTNG_ERR_INVALID;
> +               goto end;
> +       }
> +       ret = lttng_regenerate_statedump(session_name);
> +       if (ret == 0) {
> +               MSG("Statedump successfully regenerated for session %s", session_name);

Please use "State dump" in two words in human-readable messages.

> +       }
> +
> +end:
> +       return ret;
> +}
> +
>  static int handle_command(const char **argv)
>  {
>         struct cmd_struct *cmd;
> diff --git a/src/common/error.c b/src/common/error.c
> index bc42577..6730e14 100644
> --- a/src/common/error.c
> +++ b/src/common/error.c
> @@ -183,6 +183,7 @@ static const char *error_string_array[] = {
>         [ ERROR_INDEX(LTTNG_ERR_LIVE_SESSION) ] = "Live sessions are not supported",
>         [ ERROR_INDEX(LTTNG_ERR_PER_PID_SESSION) ] = "Per-PID tracing sessions are not supported",
>         [ ERROR_INDEX(LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE) ] = "Context unavailable on this kernel",
> +       [ ERROR_INDEX(LTTNG_ERR_REGEN_STATEDUMP_FAIL) ] = "Statedump regeneration failed",

"State dump"

>
>         /* Last element */
>         [ ERROR_INDEX(LTTNG_ERR_NR) ] = "Unknown error code"
> diff --git a/src/common/kernel-ctl/kernel-ctl.c b/src/common/kernel-ctl/kernel-ctl.c
> index e1de0e7..1c2ab61 100644
> --- a/src/common/kernel-ctl/kernel-ctl.c
> +++ b/src/common/kernel-ctl/kernel-ctl.c
> @@ -230,6 +230,11 @@ int kernctl_session_regenerate_metadata(int fd)
>         return LTTNG_IOCTL_CHECK(fd, LTTNG_KERNEL_SESSION_METADATA_REGEN);
>  }
>
> +int kernctl_session_regenerate_statedump(int fd)
> +{
> +       return LTTNG_IOCTL_CHECK(fd, LTTNG_KERNEL_SESSION_STATEDUMP);
> +}
> +
>  int kernctl_create_stream(int fd)
>  {
>         return compat_ioctl_no_arg(fd, LTTNG_KERNEL_OLD_STREAM,
> diff --git a/src/common/kernel-ctl/kernel-ctl.h b/src/common/kernel-ctl/kernel-ctl.h
> index f30e6f3..7c25a17 100644
> --- a/src/common/kernel-ctl/kernel-ctl.h
> +++ b/src/common/kernel-ctl/kernel-ctl.h
> @@ -66,6 +66,7 @@ int kernctl_untrack_pid(int fd, int pid);
>  int kernctl_list_tracker_pids(int fd);
>
>  int kernctl_session_regenerate_metadata(int fd);
> +int kernctl_session_regenerate_statedump(int fd);
>
>  /* Buffer operations */
>
> diff --git a/src/common/sessiond-comm/sessiond-comm.h b/src/common/sessiond-comm/sessiond-comm.h
> index 29740c1..628b1ea 100644
> --- a/src/common/sessiond-comm/sessiond-comm.h
> +++ b/src/common/sessiond-comm/sessiond-comm.h
> @@ -95,6 +95,7 @@ enum lttcomm_sessiond_command {
>         LTTNG_LIST_TRACKER_PIDS             = 34,
>         LTTNG_SET_SESSION_SHM_PATH          = 40,
>         LTTNG_REGENERATE_METADATA           = 41,
> +       LTTNG_REGENERATE_STATEDUMP          = 42,
>  };
>
>  enum lttcomm_relayd_command {
> diff --git a/src/lib/lttng-ctl/lttng-ctl.c b/src/lib/lttng-ctl/lttng-ctl.c
> index 8c78e1c..bcd59f0 100644
> --- a/src/lib/lttng-ctl/lttng-ctl.c
> +++ b/src/lib/lttng-ctl/lttng-ctl.c
> @@ -2419,6 +2419,36 @@ int lttng_metadata_regenerate(const char *session_name)
>  }
>
>  /*
> + * Regenerate the statedump of a session.
> + * Return 0 on success, a negative error code on error.
> + */
> +int lttng_regenerate_statedump(const char *session_name)
> +{
> +       int ret;
> +       struct lttcomm_session_msg lsm;
> +
> +       if (!session_name) {
> +               ret = -LTTNG_ERR_INVALID;
> +               goto end;
> +       }
> +
> +       memset(&lsm, 0, sizeof(lsm));
> +       lsm.cmd_type = LTTNG_REGENERATE_STATEDUMP;
> +
> +       lttng_ctl_copy_string(lsm.session.name, session_name,
> +                       sizeof(lsm.session.name));
> +
> +       ret = lttng_ctl_ask_sessiond(&lsm, NULL);
> +       if (ret < 0) {
> +               goto end;
> +       }
> +
> +       ret = 0;
> +end:
> +       return ret;
> +}
> +
> +/*
>   * lib constructor.
>   */
>  static void __attribute__((constructor)) init(void)
> --
> 1.9.1
>