root/daemons/controld/controld_te_callbacks.c

/* */

DEFINITIONS

1. shutdown_lock_cleared
2. te_update_diff_v1
3. process_lrm_resource_diff
4. process_resource_updates
5. extract_node_uuid
6. abort_unless_down
7. process_op_deletion
8. process_delete_diff
9. process_node_state_diff
10. process_status_diff
11. process_cib_diff
12. te_update_diff_v2
13. te_update_diff
14. process_te_message
15. cib_action_updated
16. action_timer_callback

   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/cluster.h>        /* For ONLINESTATUS etc */
  19 
  20 #include <pacemaker-controld.h>
  21 
  22 void te_update_confirm(const char *event, xmlNode * msg);
  23 
  24 #define RSC_OP_PREFIX "//" XML_TAG_DIFF_ADDED "//" XML_TAG_CIB \
  25                       "//" XML_LRM_TAG_RSC_OP "[@" XML_ATTR_ID "='"
  26 
  27 // An explicit shutdown-lock of 0 means the lock has been cleared
  28 static bool
  29 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  30 {
  31     time_t shutdown_lock = 0;
  32 
  33     return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
  34                                     &shutdown_lock) == pcmk_ok)
  35            && (shutdown_lock == 0);
  36 }
  37 
  38 static void
  39 te_update_diff_v1(const char *event, xmlNode *diff)
     /*  */
  40 {
  41     int lpc, max;
  42     xmlXPathObject *xpathObj = NULL;
  43     GString *rsc_op_xpath = NULL;
  44 
  45     CRM_CHECK(diff != NULL, return);
  46 
  47     pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
  48     controld_globals.logger_out->message(controld_globals.logger_out,
  49                                          "xml-patchset", diff);
  50 
  51     if (cib__config_changed_v1(NULL, NULL, &diff)) {
  52         abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
  53                          diff);
  54         goto bail;              /* configuration changed */
  55     }
  56 
  57     /* Tickets Attributes - Added/Updated */
  58     xpathObj =
  59         xpath_search(diff,
  60                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
  61     if (numXpathResults(xpathObj) > 0) {
  62         xmlNode *aborted = getXpathResult(xpathObj, 0);
  63 
  64         abort_transition(INFINITY, pcmk__graph_restart,
  65                          "Ticket attribute: update", aborted);
  66         goto bail;
  67 
  68     }
  69     freeXpathObject(xpathObj);
  70 
  71     /* Tickets Attributes - Removed */
  72     xpathObj =
  73         xpath_search(diff,
  74                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
  75     if (numXpathResults(xpathObj) > 0) {
  76         xmlNode *aborted = getXpathResult(xpathObj, 0);
  77 
  78         abort_transition(INFINITY, pcmk__graph_restart,
  79                          "Ticket attribute: removal", aborted);
  80         goto bail;
  81     }
  82     freeXpathObject(xpathObj);
  83 
  84     /* Transient Attributes - Removed */
  85     xpathObj =
  86         xpath_search(diff,
  87                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
  88                      XML_TAG_TRANSIENT_NODEATTRS);
  89     if (numXpathResults(xpathObj) > 0) {
  90         xmlNode *aborted = getXpathResult(xpathObj, 0);
  91 
  92         abort_transition(INFINITY, pcmk__graph_restart,
  93                          "Transient attribute: removal", aborted);
  94         goto bail;
  95 
  96     }
  97     freeXpathObject(xpathObj);
  98 
  99     // Check for lrm_resource entries
 100     xpathObj = xpath_search(diff,
 101                             "//" F_CIB_UPDATE_RESULT
 102                             "//" XML_TAG_DIFF_ADDED
 103                             "//" XML_LRM_TAG_RESOURCE);
 104     max = numXpathResults(xpathObj);
 105 
 106     /*
 107      * Updates by, or in response to, graph actions will never affect more than
 108      * one resource at a time, so such updates indicate an LRM refresh. In that
 109      * case, start a new transition rather than check each result individually,
 110      * which can result in _huge_ speedups in large clusters.
 111      *
 112      * Unfortunately, we can only do so when there are no pending actions.
 113      * Otherwise, we could mistakenly throw away those results here, and
 114      * the cluster will stall waiting for them and time out the operation.
 115      */
 116     if ((controld_globals.transition_graph->pending == 0) && (max > 1)) {
 117         crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
 118                   max);
 119         crm_log_xml_trace(diff, "lrm-refresh");
 120         abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
 121                          NULL);
 122         goto bail;
 123     }
 124 
 125     if (max == 1) {
 126         xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
 127 
 128         if (shutdown_lock_cleared(lrm_resource)) {
 129             // @TODO would be more efficient to abort once after transition done
 130             abort_transition(INFINITY, pcmk__graph_restart,
 131                              "Shutdown lock cleared", lrm_resource);
 132             // Still process results, so we stop timers and update failcounts
 133         }
 134     }
 135     freeXpathObject(xpathObj);
 136 
 137     /* Process operation updates */
 138     xpathObj =
 139         xpath_search(diff,
 140                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
 141     max = numXpathResults(xpathObj);
 142     if (max > 0) {
 143         int lpc = 0;
 144 
 145         for (lpc = 0; lpc < max; lpc++) {
 146             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 147             const char *node = get_node_id(rsc_op);
 148 
 149             process_graph_event(rsc_op, node);
 150         }
 151     }
 152     freeXpathObject(xpathObj);
 153 
 154     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
 155     xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
 156     max = numXpathResults(xpathObj);
 157     for (lpc = 0; lpc < max; lpc++) {
 158         const char *op_id = NULL;
 159         xmlXPathObject *op_match = NULL;
 160         xmlNode *match = getXpathResult(xpathObj, lpc);
 161 
 162         CRM_LOG_ASSERT(match != NULL);
 163         if(match == NULL) { continue; };
 164 
 165         op_id = ID(match);
 166 
 167         if (rsc_op_xpath == NULL) {
 168             rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
 169         } else {
 170             g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
 171         }
 172         pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);
 173 
 174         op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
 175         if (numXpathResults(op_match) == 0) {
 176             /* Prevent false positives by matching cancelations too */
 177             const char *node = get_node_id(match);
 178             pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);
 179 
 180             if (cancelled == NULL) {
 181                 crm_debug("No match for deleted action %s (%s on %s)",
 182                           (const char *) rsc_op_xpath->str, op_id, node);
 183                 abort_transition(INFINITY, pcmk__graph_restart,
 184                                  "Resource op removal", match);
 185                 freeXpathObject(op_match);
 186                 goto bail;
 187 
 188             } else {
 189                 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
 190                           op_id, node, cancelled->id);
 191             }
 192         }
 193 
 194         freeXpathObject(op_match);
 195     }
 196 
 197   bail:
 198     freeXpathObject(xpathObj);
 199     if (rsc_op_xpath != NULL) {
 200         g_string_free(rsc_op_xpath, TRUE);
 201     }
 202 }
 203 
 204 static void
 205 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
 206 {
 207     for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
 208          rsc_op = pcmk__xml_next(rsc_op)) {
 209         process_graph_event(rsc_op, node);
 210     }
 211     if (shutdown_lock_cleared(lrm_resource)) {
 212         // @TODO would be more efficient to abort once after transition done
 213         abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
 214                          lrm_resource);
 215     }
 216 }
 217 
 218 static void
 219 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
 220                          const char *op, const char *xpath)
 221 {
 222     xmlNode *rsc = NULL;
 223 
 224     if (xml == NULL) {
 225         return;
 226     }
 227 
 228     if (pcmk__xe_is(xml, XML_CIB_TAG_LRM)) {
 229         xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
 230         CRM_CHECK(xml != NULL, return);
 231     }
 232 
 233     CRM_CHECK(pcmk__xe_is(xml, XML_LRM_TAG_RESOURCES), return);
 234 
 235     /*
 236      * Updates by, or in response to, TE actions will never contain updates
 237      * for more than one resource at a time, so such updates indicate an
 238      * LRM refresh.
 239      *
 240      * In that case, start a new transition rather than check each result
 241      * individually, which can result in _huge_ speedups in large clusters.
 242      *
 243      * Unfortunately, we can only do so when there are no pending actions.
 244      * Otherwise, we could mistakenly throw away those results here, and
 245      * the cluster will stall waiting for them and time out the operation.
 246      */
 247     if ((controld_globals.transition_graph->pending == 0)
 248         && (xml->children != NULL) && (xml->children->next != NULL)) {
 249 
 250         crm_log_xml_trace(change, "lrm-refresh");
 251         abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
 252                          NULL);
 253         return;
 254     }
 255 
 256     for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
 257          rsc = pcmk__xml_next(rsc)) {
 258         crm_trace("Processing %s", ID(rsc));
 259         process_lrm_resource_diff(rsc, node);
 260     }
 261 }
 262 
 263 static char *extract_node_uuid(const char *xpath) 
     /*  */
 264 {
 265     char *mutable_path = strdup(xpath);
 266     char *node_uuid = NULL;
 267     char *search = NULL;
 268     char *match = NULL;
 269 
 270     match = strstr(mutable_path, "node_state[@" XML_ATTR_ID "=\'");
 271     if (match == NULL) {
 272         free(mutable_path);
 273         return NULL;
 274     }
 275     match += strlen("node_state[@" XML_ATTR_ID "=\'");
 276 
 277     search = strchr(match, '\'');
 278     if (search == NULL) {
 279         free(mutable_path);
 280         return NULL;
 281     }
 282     search[0] = 0;
 283 
 284     node_uuid = strdup(match);
 285     free(mutable_path);
 286     return node_uuid;
 287 }
 288 
 289 static void
 290 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 291                   const char *reason)
 292 {
 293     char *node_uuid = NULL;
 294     pcmk__graph_action_t *down = NULL;
 295 
 296     if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
 297         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 298         return;
 299     }
 300 
 301     node_uuid = extract_node_uuid(xpath);
 302     if(node_uuid == NULL) {
 303         crm_err("Could not extract node ID from %s", xpath);
 304         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 305         return;
 306     }
 307 
 308     down = match_down_event(node_uuid);
 309     if (down == NULL) {
 310         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 311         abort_transition(INFINITY, pcmk__graph_restart, reason, change);
 312     } else {
 313         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 314     }
 315     free(node_uuid);
 316 }
 317 
 318 static void
 319 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 320 {
 321     char *mutable_key = strdup(xpath);
 322     char *key;
 323     char *node_uuid;
 324 
 325     // Extract the part of xpath between last pair of single quotes
 326     key = strrchr(mutable_key, '\'');
 327     if (key != NULL) {
 328         *key = '\0';
 329         key = strrchr(mutable_key, '\'');
 330     }
 331     if (key == NULL) {
 332         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 333                  xpath);
 334         free(mutable_key);
 335         return;
 336     }
 337     ++key;
 338 
 339     node_uuid = extract_node_uuid(xpath);
 340     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 341         abort_transition(INFINITY, pcmk__graph_restart,
 342                          "Resource operation removal", change);
 343     }
 344     free(mutable_key);
 345     free(node_uuid);
 346 }
 347 
 348 static void
 349 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 350 {
 351     if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
 352         process_op_deletion(xpath, change);
 353 
 354     } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
 355         abort_unless_down(xpath, op, change, "Resource state removal");
 356 
 357     } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
 358         abort_unless_down(xpath, op, change, "Node state removal");
 359 
 360     } else {
 361         crm_trace("Ignoring delete of %s", xpath);
 362     }
 363 }
 364 
 365 static void
 366 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 367                         const char *xpath)
 368 {
 369     xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
 370 
 371     process_resource_updates(ID(state), lrm, change, op, xpath);
 372 }
 373 
 374 static void
 375 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 376                     const char *xpath)
 377 {
 378     for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
 379          state = pcmk__xml_next(state)) {
 380         process_node_state_diff(state, change, op, xpath);
 381     }
 382 }
 383 
 384 static void
 385 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 386                  const char *xpath)
 387 {
 388     xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
 389     xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
 390 
 391     if (status) {
 392         process_status_diff(status, change, op, xpath);
 393     }
 394     if (config) {
 395         abort_transition(INFINITY, pcmk__graph_restart,
 396                          "Non-status-only change", change);
 397     }
 398 }
 399 
 400 static void
 401 te_update_diff_v2(xmlNode *diff)
     /*  */
 402 {
 403     crm_log_xml_trace(diff, "Patch:Raw");
 404 
 405     for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
 406          change = pcmk__xml_next(change)) {
 407 
 408         xmlNode *match = NULL;
 409         const char *name = NULL;
 410         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 411 
 412         // Possible ops: create, modify, delete, move
 413         const char *op = crm_element_value(change, XML_DIFF_OP);
 414 
 415         // Ignore uninteresting updates
 416         if (op == NULL) {
 417             continue;
 418 
 419         } else if (xpath == NULL) {
 420             crm_trace("Ignoring %s change for version field", op);
 421             continue;
 422 
 423         } else if ((strcmp(op, "move") == 0)
 424                    && (strstr(xpath,
 425                               "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION
 426                               "/" XML_CIB_TAG_RESOURCES) == NULL)) {
 427             /* We still need to consider moves within the resources section,
 428              * since they affect placement order.
 429              */
 430             crm_trace("Ignoring move change at %s", xpath);
 431             continue;
 432         }
 433 
 434         // Find the result of create/modify ops
 435         if (strcmp(op, "create") == 0) {
 436             match = change->children;
 437 
 438         } else if (strcmp(op, "modify") == 0) {
 439             match = first_named_child(change, XML_DIFF_RESULT);
 440             if(match) {
 441                 match = match->children;
 442             }
 443 
 444         } else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
 445             crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 446                      op, xpath);
 447             continue;
 448         }
 449 
 450         if (match) {
 451             if (match->type == XML_COMMENT_NODE) {
 452                 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 453                 continue;
 454             }
 455             name = (const char *)match->name;
 456         }
 457 
 458         crm_trace("Handling %s operation for %s%s%s",
 459                   op, (xpath? xpath : "CIB"),
 460                   (name? " matched by " : ""), (name? name : ""));
 461 
 462         if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
 463             abort_transition(INFINITY, pcmk__graph_restart,
 464                              "Configuration change", change);
 465             break; // Won't be packaged with operation results we may be waiting for
 466 
 467         } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
 468                    || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
 469             abort_transition(INFINITY, pcmk__graph_restart,
 470                              "Ticket attribute change", change);
 471             break; // Won't be packaged with operation results we may be waiting for
 472 
 473         } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
 474                    || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
 475                                    pcmk__str_none)) {
 476             abort_unless_down(xpath, op, change, "Transient attribute change");
 477             break; // Won't be packaged with operation results we may be waiting for
 478 
 479         } else if (strcmp(op, "delete") == 0) {
 480             process_delete_diff(xpath, op, change);
 481 
 482         } else if (name == NULL) {
 483             crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 484                      op, xpath);
 485 
 486         } else if (strcmp(name, XML_TAG_CIB) == 0) {
 487             process_cib_diff(match, change, op, xpath);
 488 
 489         } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
 490             process_status_diff(match, change, op, xpath);
 491 
 492         } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
 493             process_node_state_diff(match, change, op, xpath);
 494 
 495         } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
 496             process_resource_updates(ID(match), match, change, op, xpath);
 497 
 498         } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
 499             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 500 
 501             process_resource_updates(local_node, match, change, op, xpath);
 502             free(local_node);
 503 
 504         } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
 505             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 506 
 507             process_lrm_resource_diff(match, local_node);
 508             free(local_node);
 509 
 510         } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
 511             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 512 
 513             process_graph_event(match, local_node);
 514             free(local_node);
 515 
 516         } else {
 517             crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 518                      op, xpath, name);
 519         }
 520     }
 521 }
 522 
 523 void
 524 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 525 {
 526     xmlNode *diff = NULL;
 527     const char *op = NULL;
 528     int rc = -EINVAL;
 529     int format = 1;
 530     int p_add[] = { 0, 0, 0 };
 531     int p_del[] = { 0, 0, 0 };
 532 
 533     CRM_CHECK(msg != NULL, return);
 534     crm_element_value_int(msg, F_CIB_RC, &rc);
 535 
 536     if (controld_globals.transition_graph == NULL) {
 537         crm_trace("No graph");
 538         return;
 539 
 540     } else if (rc < pcmk_ok) {
 541         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 542         return;
 543 
 544     } else if (controld_globals.transition_graph->complete
 545                && (controld_globals.fsa_state != S_IDLE)
 546                && (controld_globals.fsa_state != S_TRANSITION_ENGINE)
 547                && (controld_globals.fsa_state != S_POLICY_ENGINE)) {
 548         crm_trace("Filter state=%s (complete)",
 549                   fsa_state2string(controld_globals.fsa_state));
 550         return;
 551     }
 552 
 553     op = crm_element_value(msg, F_CIB_OPERATION);
 554     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 555 
 556     xml_patch_versions(diff, p_add, p_del);
 557     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 558               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 559               fsa_state2string(controld_globals.fsa_state));
 560 
 561     crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
 562     switch (format) {
 563         case 1:
 564             te_update_diff_v1(event, diff);
 565             break;
 566         case 2:
 567             te_update_diff_v2(diff);
 568             break;
 569         default:
 570             crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 571                      format);
 572     }
 573     controld_remove_all_outside_events();
 574 }
 575 
 576 void
 577 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 578 {
 579     const char *value = NULL;
 580     xmlXPathObject *xpathObj = NULL;
 581     int nmatches = 0;
 582 
 583     CRM_CHECK(msg != NULL, return);
 584 
 585     // Transition requests must specify transition engine as subsystem
 586     value = crm_element_value(msg, F_CRM_SYS_TO);
 587     if (pcmk__str_empty(value)
 588         || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
 589         crm_info("Received invalid transition request: subsystem '%s' not '"
 590                  CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
 591         return;
 592     }
 593 
 594     // Only the lrm_invoke command is supported as a transition request
 595     value = crm_element_value(msg, F_CRM_TASK);
 596     if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
 597         crm_info("Received invalid transition request: command '%s' not '"
 598                  CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
 599         return;
 600     }
 601 
 602     // Transition requests must be marked as coming from the executor
 603     value = crm_element_value(msg, F_CRM_SYS_FROM);
 604     if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
 605         crm_info("Received invalid transition request: from '%s' not '"
 606                  CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
 607         return;
 608     }
 609 
 610     crm_debug("Processing transition request with ref='%s' origin='%s'",
 611               pcmk__s(crm_element_value(msg, F_CRM_REFERENCE), ""),
 612               pcmk__s(crm_element_value(msg, F_ORIG), ""));
 613 
 614     xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
 615     nmatches = numXpathResults(xpathObj);
 616     if (nmatches == 0) {
 617         crm_err("Received transition request with no results (bug?)");
 618     } else {
 619         for (int lpc = 0; lpc < nmatches; lpc++) {
 620             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 621             const char *node = get_node_id(rsc_op);
 622 
 623             process_graph_event(rsc_op, node);
 624         }
 625     }
 626     freeXpathObject(xpathObj);
 627 }
 628 
 629 void
 630 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 631 {
 632     if (rc < pcmk_ok) {
 633         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 634     }
 635 }
 636 
 637 /*!
 638  * \brief Handle a timeout in node-to-node communication
 639  *
 640  * \param[in,out] data  Pointer to graph action
 641  *
 642  * \return FALSE (indicating that source should be not be re-added)
 643  */
 644 gboolean
 645 action_timer_callback(gpointer data)
     /*  */
 646 {
 647     pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
 648     const char *task = NULL;
 649     const char *on_node = NULL;
 650     const char *via_node = NULL;
 651 
 652     CRM_CHECK(data != NULL, return FALSE);
 653 
 654     stop_te_timer(action);
 655 
 656     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 657     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 658     via_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 659 
 660     if (controld_globals.transition_graph->complete) {
 661         crm_notice("Node %s did not send %s result (via %s) within %dms "
 662                    "(ignoring because transition not in progress)",
 663                    (on_node? on_node : ""), (task? task : "unknown action"),
 664                    (via_node? via_node : "controller"), action->timeout);
 665     } else {
 666         /* fail the action */
 667 
 668         crm_err("Node %s did not send %s result (via %s) within %dms "
 669                 "(action timeout plus cluster-delay)",
 670                 (on_node? on_node : ""), (task? task : "unknown action"),
 671                 (via_node? via_node : "controller"),
 672                 (action->timeout
 673                  + controld_globals.transition_graph->network_delay));
 674         pcmk__log_graph_action(LOG_ERR, action);
 675 
 676         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
 677 
 678         te_action_confirmed(action, controld_globals.transition_graph);
 679         abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);
 680 
 681         // Record timeout in the CIB if appropriate
 682         if ((action->type == pcmk__rsc_graph_action)
 683             && controld_action_is_recordable(task)) {
 684             controld_record_action_timeout(action);
 685         }
 686     }
 687 
 688     return FALSE;
 689 }

/* */