lib/pacemaker/pcmk_sched

/* */
This source file includes following definitions.
invert_action
get_ordering_type
get_ordering_symmetry
ordering_flags_for_kind
get_ordering_resource
get_minimum_first_instances
clone_min_ordering
inverse_ordering
unpack_simple_rsc_order
pcmk__new_ordering
unpack_order_set
order_rsc_sets
unpack_order_tags
pcmk__unpack_ordering
ordering_is_invalid
pcmk__disable_invalid_orderings
pcmk__order_stops_before_shutdown
find_actions_by_task
order_resource_actions_after
rsc_order_first
block_colocation_dependents
update_action_for_orderings
pcmk__apply_orderings
pcmk__order_after_each
pcmk__promotable_restart_ordering
   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <inttypes.h>               // PRIx32
  13 #include <stdbool.h>
  14 #include <glib.h>
  15 
  16 #include <crm/crm.h>
  17 #include <pacemaker-internal.h>
  18 #include "libpacemaker_private.h"
  19 
  20 enum pe_order_kind {
  21     pe_order_kind_optional,
  22     pe_order_kind_mandatory,
  23     pe_order_kind_serialize,
  24 };
  25 
  26 enum ordering_symmetry {
  27     ordering_asymmetric,        // the only relation in an asymmetric ordering
  28     ordering_symmetric,         // the normal relation in a symmetric ordering
  29     ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
  30 };
  31 
  32 #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {                  \
  33         __rsc = pcmk__find_constraint_resource(scheduler->resources,        \
  34                                                __name);                     \
  35         if (__rsc == NULL) {                                                \
  36             pcmk__config_err("%s: No resource found for %s", __set, __name);\
  37             return pcmk_rc_unpack_error;                                    \
  38         }                                                                   \
  39     } while (0)
  40 
  41 static const char *
  42 invert_action(const char *action)
     /*  */
  43 {
  44     if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
  45         return PCMK_ACTION_STOP;
  46 
  47     } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
  48         return PCMK_ACTION_START;
  49 
  50     } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
  51         return PCMK_ACTION_DEMOTE;
  52 
  53     } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
  54         return PCMK_ACTION_PROMOTE;
  55 
  56     } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
  57         return PCMK_ACTION_DEMOTED;
  58 
  59     } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
  60         return PCMK_ACTION_PROMOTED;
  61 
  62     } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
  63         return PCMK_ACTION_STOPPED;
  64 
  65     } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
  66         return PCMK_ACTION_RUNNING;
  67     }
  68     crm_warn("Unknown action '%s' specified in order constraint", action);
  69     return NULL;
  70 }
  71 
  72 static enum pe_order_kind
  73 get_ordering_type(const xmlNode *xml_obj)
     /*  */
  74 {
  75     enum pe_order_kind kind_e = pe_order_kind_mandatory;
  76     const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
  77 
  78     if (kind == NULL) {
  79         const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
  80 
  81         kind_e = pe_order_kind_mandatory;
  82 
  83         if (score) {
  84             // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
  85             int score_i = char2score(score);
  86 
  87             if (score_i == 0) {
  88                 kind_e = pe_order_kind_optional;
  89             }
  90             pe_warn_once(pcmk__wo_order_score,
  91                          "Support for 'score' in rsc_order is deprecated "
  92                          "and will be removed in a future release "
  93                          "(use 'kind' instead)");
  94         }
  95 
  96     } else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
  97         kind_e = pe_order_kind_mandatory;
  98 
  99     } else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
 100         kind_e = pe_order_kind_optional;
 101 
 102     } else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
 103         kind_e = pe_order_kind_serialize;
 104 
 105     } else {
 106         pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
 107                          "%s to 'Mandatory' because '%s' is not valid",
 108                          pcmk__s(ID(xml_obj), "missing ID"), kind);
 109     }
 110     return kind_e;
 111 }
 112 
 113 /*!
 114  * \internal
 115  * \brief Get ordering symmetry from XML
 116  *
 117  * \param[in] xml_obj               Ordering XML
 118  * \param[in] parent_kind           Default ordering kind
 119  * \param[in] parent_symmetrical_s  Parent element's symmetrical setting, if any
 120  *
 121  * \retval ordering_symmetric   Ordering is symmetric
 122  * \retval ordering_asymmetric  Ordering is asymmetric
 123  */
 124 static enum ordering_symmetry
 125 get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
     /*  */
 126                       const char *parent_symmetrical_s)
 127 {
 128     int rc = pcmk_rc_ok;
 129     bool symmetric = false;
 130     enum pe_order_kind kind = parent_kind; // Default to parent's kind
 131 
 132     // Check ordering XML for explicit kind
 133     if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
 134         || (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
 135         kind = get_ordering_type(xml_obj);
 136     }
 137 
 138     // Check ordering XML (and parent) for explicit symmetrical setting
 139     rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);
 140 
 141     if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
 142         symmetric = crm_is_true(parent_symmetrical_s);
 143         rc = pcmk_rc_ok;
 144     }
 145 
 146     if (rc == pcmk_rc_ok) {
 147         if (symmetric) {
 148             if (kind == pe_order_kind_serialize) {
 149                 pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
 150                                   " for '%s' because not valid with "
 151                                   XML_ORDER_ATTR_KIND " of 'Serialize'",
 152                                   ID(xml_obj));
 153             } else {
 154                 return ordering_symmetric;
 155             }
 156         }
 157         return ordering_asymmetric;
 158     }
 159 
 160     // Use default symmetry
 161     if (kind == pe_order_kind_serialize) {
 162         return ordering_asymmetric;
 163     }
 164     return ordering_symmetric;
 165 }
 166 
 167 /*!
 168  * \internal
 169  * \brief Get ordering flags appropriate to ordering kind
 170  *
 171  * \param[in] kind      Ordering kind
 172  * \param[in] first     Action name for 'first' action
 173  * \param[in] symmetry  This ordering's symmetry role
 174  *
 175  * \return Minimal ordering flags appropriate to \p kind
 176  */
 177 static uint32_t
 178 ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
     /*  */
 179                         enum ordering_symmetry symmetry)
 180 {
 181     uint32_t flags = pcmk__ar_none; // so we trace-log all flags set
 182 
 183     switch (kind) {
 184         case pe_order_kind_optional:
 185             pe__set_order_flags(flags, pcmk__ar_ordered);
 186             break;
 187 
 188         case pe_order_kind_serialize:
 189             /* This flag is not used anywhere directly but means the relation
 190              * will not match an equality comparison against pcmk__ar_none or
 191              * pcmk__ar_ordered.
 192              */
 193             pe__set_order_flags(flags, pcmk__ar_serialize);
 194             break;
 195 
 196         case pe_order_kind_mandatory:
 197             pe__set_order_flags(flags, pcmk__ar_ordered);
 198             switch (symmetry) {
 199                 case ordering_asymmetric:
 200                     pe__set_order_flags(flags, pcmk__ar_asymmetric);
 201                     break;
 202 
 203                 case ordering_symmetric:
 204                     pe__set_order_flags(flags, pcmk__ar_first_implies_then);
 205                     if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
 206                                              PCMK_ACTION_PROMOTE, NULL)) {
 207                         pe__set_order_flags(flags,
 208                                             pcmk__ar_unrunnable_first_blocks);
 209                     }
 210                     break;
 211 
 212                 case ordering_symmetric_inverse:
 213                     pe__set_order_flags(flags, pcmk__ar_then_implies_first);
 214                     break;
 215             }
 216             break;
 217     }
 218     return flags;
 219 }
 220 
 221 /*!
 222  * \internal
 223  * \brief Find resource corresponding to ID specified in ordering
 224  *
 225  * \param[in] xml            Ordering XML
 226  * \param[in] resource_attr  XML attribute name for resource ID
 227  * \param[in] instance_attr  XML attribute name for instance number.
 228  *                           This option is deprecated and will be removed in a
 229  *                           future release.
 230  * \param[in] scheduler      Scheduler data
 231  *
 232  * \return Resource corresponding to \p id, or NULL if none
 233  */
 234 static pcmk_resource_t *
 235 get_ordering_resource(const xmlNode *xml, const char *resource_attr,
     /*  */
 236                       const char *instance_attr,
 237                       const pcmk_scheduler_t *scheduler)
 238 {
 239     // @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
 240     pcmk_resource_t *rsc = NULL;
 241     const char *rsc_id = crm_element_value(xml, resource_attr);
 242     const char *instance_id = crm_element_value(xml, instance_attr);
 243 
 244     if (rsc_id == NULL) {
 245         pcmk__config_err("Ignoring constraint '%s' without %s",
 246                          ID(xml), resource_attr);
 247         return NULL;
 248     }
 249 
 250     rsc = pcmk__find_constraint_resource(scheduler->resources, rsc_id);
 251     if (rsc == NULL) {
 252         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
 253                          "does not exist", ID(xml), rsc_id);
 254         return NULL;
 255     }
 256 
 257     if (instance_id != NULL) {
 258         pe_warn_once(pcmk__wo_order_inst,
 259                      "Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
 260                      XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
 261                      "removed in a future release.");
 262 
 263         if (!pe_rsc_is_clone(rsc)) {
 264             pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
 265                              "is not a clone but instance '%s' was requested",
 266                              ID(xml), rsc_id, instance_id);
 267             return NULL;
 268         }
 269         rsc = find_clone_instance(rsc, instance_id);
 270         if (rsc == NULL) {
 271             pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
 272                              "does not have an instance '%s'",
 273                              "'%s'", ID(xml), rsc_id, instance_id);
 274             return NULL;
 275         }
 276     }
 277     return rsc;
 278 }
 279 
 280 /*!
 281  * \internal
 282  * \brief Determine minimum number of 'first' instances required in ordering
 283  *
 284  * \param[in] rsc  'First' resource in ordering
 285  * \param[in] xml  Ordering XML
 286  *
 287  * \return Minimum 'first' instances required (or 0 if not applicable)
 288  */
 289 static int
 290 get_minimum_first_instances(const pcmk_resource_t *rsc, const xmlNode *xml)
     /*  */
 291 {
 292     const char *clone_min = NULL;
 293     bool require_all = false;
 294 
 295     if (!pe_rsc_is_clone(rsc)) {
 296         return 0;
 297     }
 298 
 299     clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
 300     if (clone_min != NULL) {
 301         int clone_min_int = 0;
 302 
 303         pcmk__scan_min_int(clone_min, &clone_min_int, 0);
 304         return clone_min_int;
 305     }
 306 
 307     /* @COMPAT 1.1.13:
 308      * require-all=false is deprecated equivalent of clone-min=1
 309      */
 310     if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
 311         pe_warn_once(pcmk__wo_require_all,
 312                      "Support for require-all in ordering constraints "
 313                      "is deprecated and will be removed in a future release"
 314                      " (use clone-min clone meta-attribute instead)");
 315         if (!require_all) {
 316             return 1;
 317         }
 318     }
 319 
 320     return 0;
 321 }
 322 
 323 /*!
 324  * \internal
 325  * \brief Create orderings for a constraint with clone-min > 0
 326  *
 327  * \param[in]     id            Ordering ID
 328  * \param[in,out] rsc_first     'First' resource in ordering (a clone)
 329  * \param[in]     action_first  'First' action in ordering
 330  * \param[in]     rsc_then      'Then' resource in ordering
 331  * \param[in]     action_then   'Then' action in ordering
 332  * \param[in]     flags         Ordering flags
 333  * \param[in]     clone_min     Minimum required instances of 'first'
 334  */
 335 static void
 336 clone_min_ordering(const char *id,
     /*  */
 337                    pcmk_resource_t *rsc_first, const char *action_first,
 338                    pcmk_resource_t *rsc_then, const char *action_then,
 339                    uint32_t flags, int clone_min)
 340 {
 341     // Create a pseudo-action for when the minimum instances are active
 342     char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
 343     pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);
 344 
 345     free(task);
 346 
 347     /* Require the pseudo-action to have the required number of actions to be
 348      * considered runnable before allowing the pseudo-action to be runnable.
 349      */
 350     clone_min_met->required_runnable_before = clone_min;
 351     pe__set_action_flags(clone_min_met, pcmk_action_min_runnable);
 352 
 353     // Order the actions for each clone instance before the pseudo-action
 354     for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
 355         pcmk_resource_t *child = iter->data;
 356 
 357         pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
 358                            NULL, NULL, NULL, clone_min_met,
 359                            pcmk__ar_min_runnable
 360                            |pcmk__ar_first_implies_then_graphed,
 361                            rsc_first->cluster);
 362     }
 363 
 364     // Order "then" action after the pseudo-action (if runnable)
 365     pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
 366                        pcmk__op_key(rsc_then->id, action_then, 0),
 367                        NULL, flags|pcmk__ar_unrunnable_first_blocks,
 368                        rsc_first->cluster);
 369 }
 370 
 371 /*!
 372  * \internal
 373  * \brief Update ordering flags for restart-type=restart
 374  *
 375  * \param[in]     rsc    'Then' resource in ordering
 376  * \param[in]     kind   Ordering kind
 377  * \param[in]     flag   Ordering flag to set (when applicable)
 378  * \param[in,out] flags  Ordering flag set to update
 379  *
 380  * \compat The restart-type resource meta-attribute is deprecated. Eventually,
 381  *         it will be removed, and pe_restart_ignore will be the only behavior,
 382  *         at which time this can just be removed entirely.
 383  */
 384 #define handle_restart_type(rsc, kind, flag, flags) do {        \
 385         if (((kind) == pe_order_kind_optional)                  \
 386             && ((rsc)->restart_type == pe_restart_restart)) {   \
 387             pe__set_order_flags((flags), (flag));               \
 388         }                                                       \
 389     } while (0)
 390 
 391 /*!
 392  * \internal
 393  * \brief Create new ordering for inverse of symmetric constraint
 394  *
 395  * \param[in]     id            Ordering ID (for logging only)
 396  * \param[in]     kind          Ordering kind
 397  * \param[in]     rsc_first     'First' resource in ordering (a clone)
 398  * \param[in]     action_first  'First' action in ordering
 399  * \param[in,out] rsc_then      'Then' resource in ordering
 400  * \param[in]     action_then   'Then' action in ordering
 401  */
 402 static void
 403 inverse_ordering(const char *id, enum pe_order_kind kind,
     /*  */
 404                  pcmk_resource_t *rsc_first, const char *action_first,
 405                  pcmk_resource_t *rsc_then, const char *action_then)
 406 {
 407     action_then = invert_action(action_then);
 408     action_first = invert_action(action_first);
 409     if ((action_then == NULL) || (action_first == NULL)) {
 410         pcmk__config_warn("Cannot invert constraint '%s' "
 411                           "(please specify inverse manually)", id);
 412     } else {
 413         uint32_t flags = ordering_flags_for_kind(kind, action_first,
 414                                                  ordering_symmetric_inverse);
 415 
 416         handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
 417         pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
 418                                      action_first, flags);
 419     }
 420 }
 421 
 422 static void
 423 unpack_simple_rsc_order(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
     /*  */
 424 {
 425     pcmk_resource_t *rsc_then = NULL;
 426     pcmk_resource_t *rsc_first = NULL;
 427     int min_required_before = 0;
 428     enum pe_order_kind kind = pe_order_kind_mandatory;
 429     uint32_t flags = pcmk__ar_none;
 430     enum ordering_symmetry symmetry;
 431 
 432     const char *action_then = NULL;
 433     const char *action_first = NULL;
 434     const char *id = NULL;
 435 
 436     CRM_CHECK(xml_obj != NULL, return);
 437 
 438     id = crm_element_value(xml_obj, XML_ATTR_ID);
 439     if (id == NULL) {
 440         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
 441                          xml_obj->name);
 442         return;
 443     }
 444 
 445     rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
 446                                       XML_ORDER_ATTR_FIRST_INSTANCE,
 447                                       scheduler);
 448     if (rsc_first == NULL) {
 449         return;
 450     }
 451 
 452     rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
 453                                      XML_ORDER_ATTR_THEN_INSTANCE,
 454                                      scheduler);
 455     if (rsc_then == NULL) {
 456         return;
 457     }
 458 
 459     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
 460     if (action_first == NULL) {
 461         action_first = PCMK_ACTION_START;
 462     }
 463 
 464     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
 465     if (action_then == NULL) {
 466         action_then = action_first;
 467     }
 468 
 469     kind = get_ordering_type(xml_obj);
 470 
 471     symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
 472     flags = ordering_flags_for_kind(kind, action_first, symmetry);
 473 
 474     handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);
 475 
 476     /* If there is a minimum number of instances that must be runnable before
 477      * the 'then' action is runnable, we use a pseudo-action for convenience:
 478      * minimum number of clone instances have runnable actions ->
 479      * pseudo-action is runnable -> dependency is runnable.
 480      */
 481     min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
 482     if (min_required_before > 0) {
 483         clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
 484                            flags, min_required_before);
 485     } else {
 486         pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
 487                                      action_then, flags);
 488     }
 489 
 490     if (symmetry == ordering_symmetric) {
 491         inverse_ordering(id, kind, rsc_first, action_first,
 492                          rsc_then, action_then);
 493     }
 494 }
 495 
 496 /*!
 497  * \internal
 498  * \brief Create a new ordering between two actions
 499  *
 500  * \param[in,out] first_rsc          Resource for 'first' action (if NULL and
 501  *                                   \p first_action is a resource action, that
 502  *                                   resource will be used)
 503  * \param[in,out] first_action_task  Action key for 'first' action (if NULL and
 504  *                                   \p first_action is not NULL, its UUID will
 505  *                                   be used)
 506  * \param[in,out] first_action       'first' action (if NULL, \p first_rsc and
 507  *                                   \p first_action_task must be set)
 508  *
 509  * \param[in]     then_rsc           Resource for 'then' action (if NULL and
 510  *                                   \p then_action is a resource action, that
 511  *                                   resource will be used)
 512  * \param[in,out] then_action_task   Action key for 'then' action (if NULL and
 513  *                                   \p then_action is not NULL, its UUID will
 514  *                                   be used)
 515  * \param[in]     then_action        'then' action (if NULL, \p then_rsc and
 516  *                                   \p then_action_task must be set)
 517  *
 518  * \param[in]     flags              Group of enum pcmk__action_relation_flags
 519  * \param[in,out] sched              Scheduler data to add ordering to
 520  *
 521  * \note This function takes ownership of first_action_task and
 522  *       then_action_task, which do not need to be freed by the caller.
 523  */
 524 void
 525 pcmk__new_ordering(pcmk_resource_t *first_rsc, char *first_action_task,
     /*  */
 526                    pcmk_action_t *first_action, pcmk_resource_t *then_rsc,
 527                    char *then_action_task, pcmk_action_t *then_action,
 528                    uint32_t flags, pcmk_scheduler_t *sched)
 529 {
 530     pe__ordering_t *order = NULL;
 531 
 532     // One of action or resource must be specified for each side
 533     CRM_CHECK(((first_action != NULL) || (first_rsc != NULL))
 534               && ((then_action != NULL) || (then_rsc != NULL)),
 535               free(first_action_task); free(then_action_task); return);
 536 
 537     if ((first_rsc == NULL) && (first_action != NULL)) {
 538         first_rsc = first_action->rsc;
 539     }
 540     if ((then_rsc == NULL) && (then_action != NULL)) {
 541         then_rsc = then_action->rsc;
 542     }
 543 
 544     order = calloc(1, sizeof(pe__ordering_t));
 545     CRM_ASSERT(order != NULL);
 546 
 547     order->id = sched->order_id++;
 548     order->flags = flags;
 549     order->lh_rsc = first_rsc;
 550     order->rh_rsc = then_rsc;
 551     order->lh_action = first_action;
 552     order->rh_action = then_action;
 553     order->lh_action_task = first_action_task;
 554     order->rh_action_task = then_action_task;
 555 
 556     if ((order->lh_action_task == NULL) && (first_action != NULL)) {
 557         order->lh_action_task = strdup(first_action->uuid);
 558     }
 559 
 560     if ((order->rh_action_task == NULL) && (then_action != NULL)) {
 561         order->rh_action_task = strdup(then_action->uuid);
 562     }
 563 
 564     if ((order->lh_rsc == NULL) && (first_action != NULL)) {
 565         order->lh_rsc = first_action->rsc;
 566     }
 567 
 568     if ((order->rh_rsc == NULL) && (then_action != NULL)) {
 569         order->rh_rsc = then_action->rsc;
 570     }
 571 
 572     pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
 573                  (sched->order_id - 1),
 574                  pcmk__s(order->lh_action_task, "an underspecified action"),
 575                  pcmk__s(order->rh_action_task, "an underspecified action"));
 576 
 577     sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
 578                                                  order);
 579     pcmk__order_migration_equivalents(order);
 580 }
 581 
 582 /*!
 583  * \brief Unpack a set in an ordering constraint
 584  *
 585  * \param[in]     set                   Set XML to unpack
 586  * \param[in]     parent_kind           rsc_order XML "kind" attribute
 587  * \param[in]     parent_symmetrical_s  rsc_order XML "symmetrical" attribute
 588  * \param[in,out] scheduler             Scheduler data
 589  *
 590  * \return Standard Pacemaker return code
 591  */
 592 static int
 593 unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
     /*  */
 594                  const char *parent_symmetrical_s, pcmk_scheduler_t *scheduler)
 595 {
 596     GList *set_iter = NULL;
 597     GList *resources = NULL;
 598 
 599     pcmk_resource_t *last = NULL;
 600     pcmk_resource_t *resource = NULL;
 601 
 602     int local_kind = parent_kind;
 603     bool sequential = false;
 604     uint32_t flags = pcmk__ar_ordered;
 605     enum ordering_symmetry symmetry;
 606 
 607     char *key = NULL;
 608     const char *id = ID(set);
 609     const char *action = crm_element_value(set, "action");
 610     const char *sequential_s = crm_element_value(set, "sequential");
 611     const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);
 612 
 613     if (action == NULL) {
 614         action = PCMK_ACTION_START;
 615     }
 616 
 617     if (kind_s) {
 618         local_kind = get_ordering_type(set);
 619     }
 620     if (sequential_s == NULL) {
 621         sequential_s = "1";
 622     }
 623 
 624     sequential = crm_is_true(sequential_s);
 625 
 626     symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
 627     flags = ordering_flags_for_kind(local_kind, action, symmetry);
 628 
 629     for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
 630          xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 631 
 632         EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
 633         resources = g_list_append(resources, resource);
 634     }
 635 
 636     if (pcmk__list_of_1(resources)) {
 637         crm_trace("Single set: %s", id);
 638         goto done;
 639     }
 640 
 641     set_iter = resources;
 642     while (set_iter != NULL) {
 643         resource = (pcmk_resource_t *) set_iter->data;
 644         set_iter = set_iter->next;
 645 
 646         key = pcmk__op_key(resource->id, action, 0);
 647 
 648         if (local_kind == pe_order_kind_serialize) {
 649             /* Serialize before everything that comes after */
 650 
 651             for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
 652                 pcmk_resource_t *then_rsc = iter->data;
 653                 char *then_key = pcmk__op_key(then_rsc->id, action, 0);
 654 
 655                 pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
 656                                    then_key, NULL, flags, scheduler);
 657             }
 658 
 659         } else if (sequential) {
 660             if (last != NULL) {
 661                 pcmk__order_resource_actions(last, action, resource, action,
 662                                              flags);
 663             }
 664             last = resource;
 665         }
 666         free(key);
 667     }
 668 
 669     if (symmetry == ordering_asymmetric) {
 670         goto done;
 671     }
 672 
 673     last = NULL;
 674     action = invert_action(action);
 675 
 676     flags = ordering_flags_for_kind(local_kind, action,
 677                                     ordering_symmetric_inverse);
 678 
 679     set_iter = resources;
 680     while (set_iter != NULL) {
 681         resource = (pcmk_resource_t *) set_iter->data;
 682         set_iter = set_iter->next;
 683 
 684         if (sequential) {
 685             if (last != NULL) {
 686                 pcmk__order_resource_actions(resource, action, last, action,
 687                                              flags);
 688             }
 689             last = resource;
 690         }
 691     }
 692 
 693   done:
 694     g_list_free(resources);
 695     return pcmk_rc_ok;
 696 }
 697 
 698 /*!
 699  * \brief Order two resource sets relative to each other
 700  *
 701  * \param[in]     id         Ordering ID (for logging)
 702  * \param[in]     set1       First listed set
 703  * \param[in]     set2       Second listed set
 704  * \param[in]     kind       Ordering kind
 705  * \param[in,out] scheduler  Scheduler data
 706  * \param[in]     symmetry   Which ordering symmetry applies to this relation
 707  *
 708  * \return Standard Pacemaker return code
 709  */
 710 static int
 711 order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2,
     /*  */
 712                enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
 713                enum ordering_symmetry symmetry)
 714 {
 715 
 716     const xmlNode *xml_rsc = NULL;
 717     const xmlNode *xml_rsc_2 = NULL;
 718 
 719     pcmk_resource_t *rsc_1 = NULL;
 720     pcmk_resource_t *rsc_2 = NULL;
 721 
 722     const char *action_1 = crm_element_value(set1, "action");
 723     const char *action_2 = crm_element_value(set2, "action");
 724 
 725     uint32_t flags = pcmk__ar_none;
 726 
 727     bool require_all = true;
 728 
 729     (void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);
 730 
 731     if (action_1 == NULL) {
 732         action_1 = PCMK_ACTION_START;
 733     }
 734 
 735     if (action_2 == NULL) {
 736         action_2 = PCMK_ACTION_START;
 737     }
 738 
 739     if (symmetry == ordering_symmetric_inverse) {
 740         action_1 = invert_action(action_1);
 741         action_2 = invert_action(action_2);
 742     }
 743 
 744     if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
 745         || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
 746         /* Assuming: A -> ( B || C) -> D
 747          * The one-or-more logic only applies during the start/promote phase.
 748          * During shutdown neither B nor can shutdown until D is down, so simply
 749          * turn require_all back on.
 750          */
 751         require_all = true;
 752     }
 753 
 754     flags = ordering_flags_for_kind(kind, action_1, symmetry);
 755 
 756     /* If we have an unordered set1, whether it is sequential or not is
 757      * irrelevant in regards to set2.
 758      */
 759     if (!require_all) {
 760         char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
 761         pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);
 762 
 763         free(task);
 764         pe__set_action_flags(unordered_action, pcmk_action_min_runnable);
 765 
 766         for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
 767              xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 768 
 769             EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 770 
 771             /* Add an ordering constraint between every element in set1 and the
 772              * pseudo action. If any action in set1 is runnable the pseudo
 773              * action will be runnable.
 774              */
 775             pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
 776                                NULL, NULL, NULL, unordered_action,
 777                                pcmk__ar_min_runnable
 778                                |pcmk__ar_first_implies_then_graphed,
 779                                scheduler);
 780         }
 781         for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
 782              xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
 783 
 784             EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
 785 
 786             /* Add an ordering constraint between the pseudo-action and every
 787              * element in set2. If the pseudo-action is runnable, every action
 788              * in set2 will be runnable.
 789              */
 790             pcmk__new_ordering(NULL, NULL, unordered_action,
 791                                rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
 792                                NULL, flags|pcmk__ar_unrunnable_first_blocks,
 793                                scheduler);
 794         }
 795 
 796         return pcmk_rc_ok;
 797     }
 798 
 799     if (pcmk__xe_attr_is_true(set1, "sequential")) {
 800         if (symmetry == ordering_symmetric_inverse) {
 801             // Get the first one
 802             xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
 803             if (xml_rsc != NULL) {
 804                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 805             }
 806 
 807         } else {
 808             // Get the last one
 809             const char *rid = NULL;
 810 
 811             for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
 812                  xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 813 
 814                 rid = ID(xml_rsc);
 815             }
 816             EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
 817         }
 818     }
 819 
 820     if (pcmk__xe_attr_is_true(set2, "sequential")) {
 821         if (symmetry == ordering_symmetric_inverse) {
 822             // Get the last one
 823             const char *rid = NULL;
 824 
 825             for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
 826                  xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 827 
 828                 rid = ID(xml_rsc);
 829             }
 830             EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
 831 
 832         } else {
 833             // Get the first one
 834             xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
 835             if (xml_rsc != NULL) {
 836                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
 837             }
 838         }
 839     }
 840 
 841     if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
 842         pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);
 843 
 844     } else if (rsc_1 != NULL) {
 845         for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
 846              xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 847 
 848             EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
 849             pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
 850                                          flags);
 851         }
 852 
 853     } else if (rsc_2 != NULL) {
 854         for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
 855              xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 856 
 857             EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 858             pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
 859                                          flags);
 860         }
 861 
 862     } else {
 863         for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
 864              xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 865 
 866             EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 867 
 868             for (xmlNode *xml_rsc_2 = first_named_child(set2,
 869                                                         XML_TAG_RESOURCE_REF);
 870                  xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
 871 
 872                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
 873                 pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
 874                                              action_2, flags);
 875             }
 876         }
 877     }
 878 
 879     return pcmk_rc_ok;
 880 }
 881 
 882 /*!
 883  * \internal
 884  * \brief If an ordering constraint uses resource tags, expand them
 885  *
 886  * \param[in,out] xml_obj       Ordering constraint XML
 887  * \param[out]    expanded_xml  Equivalent XML with tags expanded
 888  * \param[in]     scheduler     Scheduler data
 889  *
 890  * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
 891  *         and pcmk_rc_unpack_error on invalid configuration)
 892  */
 893 static int
 894 unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml,
     /*  */
 895                   const pcmk_scheduler_t *scheduler)
 896 {
 897     const char *id_first = NULL;
 898     const char *id_then = NULL;
 899     const char *action_first = NULL;
 900     const char *action_then = NULL;
 901 
 902     pcmk_resource_t *rsc_first = NULL;
 903     pcmk_resource_t *rsc_then = NULL;
 904     pcmk_tag_t *tag_first = NULL;
 905     pcmk_tag_t *tag_then = NULL;
 906 
 907     xmlNode *rsc_set_first = NULL;
 908     xmlNode *rsc_set_then = NULL;
 909     bool any_sets = false;
 910 
 911     // Check whether there are any resource sets with template or tag references
 912     *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
 913     if (*expanded_xml != NULL) {
 914         crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
 915         return pcmk_rc_ok;
 916     }
 917 
 918     id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
 919     id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
 920     if ((id_first == NULL) || (id_then == NULL)) {
 921         return pcmk_rc_ok;
 922     }
 923 
 924     if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
 925                                      &tag_first)) {
 926         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
 927                          "valid resource or tag", ID(xml_obj), id_first);
 928         return pcmk_rc_unpack_error;
 929     }
 930 
 931     if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
 932                                      &tag_then)) {
 933         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
 934                          "valid resource or tag", ID(xml_obj), id_then);
 935         return pcmk_rc_unpack_error;
 936     }
 937 
 938     if ((rsc_first != NULL) && (rsc_then != NULL)) {
 939         // Neither side references a template or tag
 940         return pcmk_rc_ok;
 941     }
 942 
 943     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
 944     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
 945 
 946     *expanded_xml = copy_xml(xml_obj);
 947 
 948     // Convert template/tag reference in "first" into constraint resource_set
 949     if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
 950                           true, scheduler)) {
 951         free_xml(*expanded_xml);
 952         *expanded_xml = NULL;
 953         return pcmk_rc_unpack_error;
 954     }
 955 
 956     if (rsc_set_first != NULL) {
 957         if (action_first != NULL) {
 958             // Move "first-action" into converted resource_set as "action"
 959             crm_xml_add(rsc_set_first, "action", action_first);
 960             xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
 961         }
 962         any_sets = true;
 963     }
 964 
 965     // Convert template/tag reference in "then" into constraint resource_set
 966     if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
 967                           true, scheduler)) {
 968         free_xml(*expanded_xml);
 969         *expanded_xml = NULL;
 970         return pcmk_rc_unpack_error;
 971     }
 972 
 973     if (rsc_set_then != NULL) {
 974         if (action_then != NULL) {
 975             // Move "then-action" into converted resource_set as "action"
 976             crm_xml_add(rsc_set_then, "action", action_then);
 977             xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
 978         }
 979         any_sets = true;
 980     }
 981 
 982     if (any_sets) {
 983         crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
 984     } else {
 985         free_xml(*expanded_xml);
 986         *expanded_xml = NULL;
 987     }
 988 
 989     return pcmk_rc_ok;
 990 }
 991 
 992 /*!
 993  * \internal
 994  * \brief Unpack ordering constraint XML
 995  *
 996  * \param[in,out] xml_obj    Ordering constraint XML to unpack
 997  * \param[in,out] scheduler  Scheduler data
 998  */
 999 void
1000 pcmk__unpack_ordering(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
     /*  */
1001 {
1002     xmlNode *set = NULL;
1003     xmlNode *last = NULL;
1004 
1005     xmlNode *orig_xml = NULL;
1006     xmlNode *expanded_xml = NULL;
1007 
1008     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
1009     const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
1010     enum pe_order_kind kind = get_ordering_type(xml_obj);
1011 
1012     enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
1013                                                             NULL);
1014 
1015     // Expand any resource tags in the constraint XML
1016     if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
1017         return;
1018     }
1019     if (expanded_xml != NULL) {
1020         orig_xml = xml_obj;
1021         xml_obj = expanded_xml;
1022     }
1023 
1024     // If the constraint has resource sets, unpack them
1025     for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
1026          set != NULL; set = crm_next_same_xml(set)) {
1027 
1028         set = expand_idref(set, scheduler->input);
1029         if ((set == NULL) // Configuration error, message already logged
1030             || (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {
1031 
1032             if (expanded_xml != NULL) {
1033                 free_xml(expanded_xml);
1034             }
1035             return;
1036         }
1037 
1038         if (last != NULL) {
1039 
1040             if (order_rsc_sets(id, last, set, kind, scheduler,
1041                                symmetry) != pcmk_rc_ok) {
1042                 if (expanded_xml != NULL) {
1043                     free_xml(expanded_xml);
1044                 }
1045                 return;
1046             }
1047 
1048             if ((symmetry == ordering_symmetric)
1049                 && (order_rsc_sets(id, set, last, kind, scheduler,
1050                                    ordering_symmetric_inverse) != pcmk_rc_ok)) {
1051                 if (expanded_xml != NULL) {
1052                     free_xml(expanded_xml);
1053                 }
1054                 return;
1055             }
1056 
1057         }
1058         last = set;
1059     }
1060 
1061     if (expanded_xml) {
1062         free_xml(expanded_xml);
1063         xml_obj = orig_xml;
1064     }
1065 
1066     // If the constraint has no resource sets, unpack it as a simple ordering
1067     if (last == NULL) {
1068         return unpack_simple_rsc_order(xml_obj, scheduler);
1069     }
1070 }
1071 
1072 static bool
1073 ordering_is_invalid(pcmk_action_t *action, pcmk__related_action_t *input)
     /*  */
1074 {
1075     /* Prevent user-defined ordering constraints between resources
1076      * running in a guest node and the resource that defines that node.
1077      */
1078     if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed)
1079         && (input->action->rsc != NULL)
1080         && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {
1081 
1082         crm_warn("Invalid ordering constraint between %s and %s",
1083                  input->action->rsc->id, action->rsc->id);
1084         return true;
1085     }
1086 
1087     /* If there's an order like
1088      * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
1089      *
1090      * then rscA is being migrated from node1 to node2, while rscB is being
1091      * migrated from node2 to node1. If there would be a graph loop,
1092      * break the order "load_stopped_node2" -> "rscA_migrate_to node1".
1093      */
1094     if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target)
1095         && (action->rsc != NULL)
1096         && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
1097         && pcmk__graph_has_loop(action, action, input)) {
1098         return true;
1099     }
1100 
1101     return false;
1102 }
1103 
1104 void
1105 pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
     /*  */
1106 {
1107     for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
1108         pcmk_action_t *action = (pcmk_action_t *) iter->data;
1109         pcmk__related_action_t *input = NULL;
1110 
1111         for (GList *input_iter = action->actions_before;
1112              input_iter != NULL; input_iter = input_iter->next) {
1113 
1114             input = input_iter->data;
1115             if (ordering_is_invalid(action, input)) {
1116                 input->type = (enum pe_ordering) pcmk__ar_none;
1117             }
1118         }
1119     }
1120 }
1121 
1122 /*!
1123  * \internal
1124  * \brief Order stops on a node before the node's shutdown
1125  *
1126  * \param[in,out] node         Node being shut down
1127  * \param[in]     shutdown_op  Shutdown action for node
1128  */
1129 void
1130 pcmk__order_stops_before_shutdown(pcmk_node_t *node, pcmk_action_t *shutdown_op)
     /*  */
1131 {
1132     for (GList *iter = node->details->data_set->actions;
1133          iter != NULL; iter = iter->next) {
1134 
1135         pcmk_action_t *action = (pcmk_action_t *) iter->data;
1136 
1137         // Only stops on the node shutting down are relevant
1138         if (!pe__same_node(action->node, node)
1139             || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
1140             continue;
1141         }
1142 
1143         // Resources and nodes in maintenance mode won't be touched
1144 
1145         if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
1146             pe_rsc_trace(action->rsc,
1147                          "Not ordering %s before shutdown of %s because "
1148                          "resource in maintenance mode",
1149                          action->uuid, pe__node_name(node));
1150             continue;
1151 
1152         } else if (node->details->maintenance) {
1153             pe_rsc_trace(action->rsc,
1154                          "Not ordering %s before shutdown of %s because "
1155                          "node in maintenance mode",
1156                          action->uuid, pe__node_name(node));
1157             continue;
1158         }
1159 
1160         /* Don't touch a resource that is unmanaged or blocked, to avoid
1161          * blocking the shutdown (though if another action depends on this one,
1162          * we may still end up blocking)
1163          */
1164         if (!pcmk_any_flags_set(action->rsc->flags,
1165                                 pcmk_rsc_managed|pcmk_rsc_blocked)) {
1166             pe_rsc_trace(action->rsc,
1167                          "Not ordering %s before shutdown of %s because "
1168                          "resource is unmanaged or blocked",
1169                          action->uuid, pe__node_name(node));
1170             continue;
1171         }
1172 
1173         pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
1174                      action->uuid, pe__node_name(node));
1175         pe__clear_action_flags(action, pcmk_action_optional);
1176         pcmk__new_ordering(action->rsc, NULL, action, NULL,
1177                            strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
1178                            pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks,
1179                            node->details->data_set);
1180     }
1181 }
1182 
1183 /*!
1184  * \brief Find resource actions matching directly or as child
1185  *
1186  * \param[in] rsc           Resource to check
1187  * \param[in] original_key  Action key to search for (possibly referencing
1188  *                          parent of \rsc)
1189  *
1190  * \return Newly allocated list of matching actions
1191  * \note It is the caller's responsibility to free the result with g_list_free()
1192  */
1193 static GList *
1194 find_actions_by_task(const pcmk_resource_t *rsc, const char *original_key)
     /*  */
1195 {
1196     // Search under given task key directly
1197     GList *list = find_actions(rsc->actions, original_key, NULL);
1198 
1199     if (list == NULL) {
1200         // Search again using this resource's ID
1201         char *key = NULL;
1202         char *task = NULL;
1203         guint interval_ms = 0;
1204 
1205         if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
1206             key = pcmk__op_key(rsc->id, task, interval_ms);
1207             list = find_actions(rsc->actions, key, NULL);
1208             free(key);
1209             free(task);
1210         } else {
1211             crm_err("Invalid operation key (bug?): %s", original_key);
1212         }
1213     }
1214     return list;
1215 }
1216 
1217 /*!
1218  * \internal
1219  * \brief Order relevant resource actions after a given action
1220  *
1221  * \param[in,out] first_action  Action to order after (or NULL if none runnable)
1222  * \param[in]     rsc           Resource whose actions should be ordered
1223  * \param[in,out] order         Ordering constraint being applied
1224  */
1225 static void
1226 order_resource_actions_after(pcmk_action_t *first_action,
     /*  */
1227                              const pcmk_resource_t *rsc, pe__ordering_t *order)
1228 {
1229     GList *then_actions = NULL;
1230     uint32_t flags = pcmk__ar_none;
1231 
1232     CRM_CHECK((rsc != NULL) && (order != NULL), return);
1233 
1234     flags = order->flags;
1235     pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
1236                  order->id, rsc->id);
1237 
1238     if (order->rh_action != NULL) {
1239         then_actions = g_list_prepend(NULL, order->rh_action);
1240 
1241     } else {
1242         then_actions = find_actions_by_task(rsc, order->rh_action_task);
1243     }
1244 
1245     if (then_actions == NULL) {
1246         pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
1247                      order->id, order->rh_action_task, rsc->id);
1248         return;
1249     }
1250 
1251     if ((first_action != NULL) && (first_action->rsc == rsc)
1252         && pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {
1253 
1254         pe_rsc_trace(rsc,
1255                      "Detected dangling migration ordering (%s then %s %s)",
1256                      first_action->uuid, order->rh_action_task, rsc->id);
1257         pe__clear_order_flags(flags, pcmk__ar_first_implies_then);
1258     }
1259 
1260     if ((first_action == NULL)
1261         && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {
1262 
1263         pe_rsc_debug(rsc,
1264                      "Ignoring ordering %d for %s: No first action found",
1265                      order->id, rsc->id);
1266         g_list_free(then_actions);
1267         return;
1268     }
1269 
1270     for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
1271         pcmk_action_t *then_action_iter = (pcmk_action_t *) iter->data;
1272 
1273         if (first_action != NULL) {
1274             order_actions(first_action, then_action_iter, flags);
1275         } else {
1276             pe__clear_action_flags(then_action_iter, pcmk_action_runnable);
1277             crm_warn("%s of %s is unrunnable because there is no %s of %s "
1278                      "to order it after", then_action_iter->task, rsc->id,
1279                      order->lh_action_task, order->lh_rsc->id);
1280         }
1281     }
1282 
1283     g_list_free(then_actions);
1284 }
1285 
1286 static void
1287 rsc_order_first(pcmk_resource_t *first_rsc, pe__ordering_t *order)
     /*  */
1288 {
1289     GList *first_actions = NULL;
1290     pcmk_action_t *first_action = order->lh_action;
1291     pcmk_resource_t *then_rsc = order->rh_rsc;
1292 
1293     CRM_ASSERT(first_rsc != NULL);
1294     pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
1295                  order->id, first_rsc->id);
1296 
1297     if (first_action != NULL) {
1298         first_actions = g_list_prepend(NULL, first_action);
1299 
1300     } else {
1301         first_actions = find_actions_by_task(first_rsc, order->lh_action_task);
1302     }
1303 
1304     if ((first_actions == NULL) && (first_rsc == then_rsc)) {
1305         pe_rsc_trace(first_rsc,
1306                      "Ignoring constraint %d: first (%s for %s) not found",
1307                      order->id, order->lh_action_task, first_rsc->id);
1308 
1309     } else if (first_actions == NULL) {
1310         char *key = NULL;
1311         char *op_type = NULL;
1312         guint interval_ms = 0;
1313 
1314         parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
1315         key = pcmk__op_key(first_rsc->id, op_type, interval_ms);
1316 
1317         if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
1318             && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
1319             free(key);
1320             pe_rsc_trace(first_rsc,
1321                          "Ignoring constraint %d: first (%s for %s) not found",
1322                          order->id, order->lh_action_task, first_rsc->id);
1323 
1324         } else if ((first_rsc->fns->state(first_rsc,
1325                                           TRUE) == pcmk_role_unpromoted)
1326                    && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
1327                                    pcmk__str_none)) {
1328             free(key);
1329             pe_rsc_trace(first_rsc,
1330                          "Ignoring constraint %d: first (%s for %s) not found",
1331                          order->id, order->lh_action_task, first_rsc->id);
1332 
1333         } else {
1334             pe_rsc_trace(first_rsc,
1335                          "Creating first (%s for %s) for constraint %d ",
1336                          order->lh_action_task, first_rsc->id, order->id);
1337             first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
1338                                          first_rsc->cluster);
1339             first_actions = g_list_prepend(NULL, first_action);
1340         }
1341 
1342         free(op_type);
1343     }
1344 
1345     if (then_rsc == NULL) {
1346         if (order->rh_action == NULL) {
1347             pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
1348                          order->id);
1349             return;
1350         }
1351         then_rsc = order->rh_action->rsc;
1352     }
1353     for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
1354         first_action = iter->data;
1355 
1356         if (then_rsc == NULL) {
1357             order_actions(first_action, order->rh_action, order->flags);
1358 
1359         } else {
1360             order_resource_actions_after(first_action, then_rsc, order);
1361         }
1362     }
1363 
1364     g_list_free(first_actions);
1365 }
1366 
1367 // GFunc to call pcmk__block_colocation_dependents()
1368 static void
1369 block_colocation_dependents(gpointer data, gpointer user_data)
     /*  */
1370 {
1371     pcmk__block_colocation_dependents(data);
1372 }
1373 
1374 // GFunc to call pcmk__update_action_for_orderings()
1375 static void
1376 update_action_for_orderings(gpointer data, gpointer user_data)
     /*  */
1377 {
1378     pcmk__update_action_for_orderings((pcmk_action_t *) data,
1379                                       (pcmk_scheduler_t *) user_data);
1380 }
1381 
1382 /*!
1383  * \internal
1384  * \brief Apply all ordering constraints
1385  *
1386  * \param[in,out] sched  Scheduler data
1387  */
1388 void
1389 pcmk__apply_orderings(pcmk_scheduler_t *sched)
     /*  */
1390 {
1391     crm_trace("Applying ordering constraints");
1392 
1393     /* Ordering constraints need to be processed in the order they were created.
1394      * rsc_order_first() and order_resource_actions_after() require the relevant
1395      * actions to already exist in some cases, but rsc_order_first() will create
1396      * the 'first' action in certain cases. Thus calling rsc_order_first() can
1397      * change the behavior of later-created orderings.
1398      *
1399      * Also, g_list_append() should be avoided for performance reasons, so we
1400      * prepend orderings when creating them and reverse the list here.
1401      *
1402      * @TODO This is brittle and should be carefully redesigned so that the
1403      * order of creation doesn't matter, and the reverse becomes unneeded.
1404      */
1405     sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);
1406 
1407     for (GList *iter = sched->ordering_constraints;
1408          iter != NULL; iter = iter->next) {
1409 
1410         pe__ordering_t *order = iter->data;
1411         pcmk_resource_t *rsc = order->lh_rsc;
1412 
1413         if (rsc != NULL) {
1414             rsc_order_first(rsc, order);
1415             continue;
1416         }
1417 
1418         rsc = order->rh_rsc;
1419         if (rsc != NULL) {
1420             order_resource_actions_after(order->lh_action, rsc, order);
1421 
1422         } else {
1423             crm_trace("Applying ordering constraint %d (non-resource actions)",
1424                       order->id);
1425             order_actions(order->lh_action, order->rh_action, order->flags);
1426         }
1427     }
1428 
1429     g_list_foreach(sched->actions, block_colocation_dependents, NULL);
1430 
1431     crm_trace("Ordering probes");
1432     pcmk__order_probes(sched);
1433 
1434     crm_trace("Updating %d actions", g_list_length(sched->actions));
1435     g_list_foreach(sched->actions, update_action_for_orderings, sched);
1436 
1437     pcmk__disable_invalid_orderings(sched);
1438 }
1439 
1440 /*!
1441  * \internal
1442  * \brief Order a given action after each action in a given list
1443  *
1444  * \param[in,out] after  "After" action
1445  * \param[in,out] list   List of "before" actions
1446  */
1447 void
1448 pcmk__order_after_each(pcmk_action_t *after, GList *list)
     /*  */
1449 {
1450     const char *after_desc = (after->task == NULL)? after->uuid : after->task;
1451 
1452     for (GList *iter = list; iter != NULL; iter = iter->next) {
1453         pcmk_action_t *before = (pcmk_action_t *) iter->data;
1454         const char *before_desc = before->task? before->task : before->uuid;
1455 
1456         crm_debug("Ordering %s on %s before %s on %s",
1457                   before_desc, pe__node_name(before->node),
1458                   after_desc, pe__node_name(after->node));
1459         order_actions(before, after, pcmk__ar_ordered);
1460     }
1461 }
1462 
1463 /*!
1464  * \internal
1465  * \brief Order promotions and demotions for restarts of a clone or bundle
1466  *
1467  * \param[in,out] rsc  Clone or bundle to order
1468  */
1469 void
1470 pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
     /*  */
1471 {
1472     // Order start and promote after all instances are stopped
1473     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1474                                  rsc, PCMK_ACTION_START,
1475                                  pcmk__ar_ordered);
1476     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
1477                                  rsc, PCMK_ACTION_PROMOTE,
1478                                  pcmk__ar_ordered);
1479 
1480     // Order stop, start, and promote after all instances are demoted
1481     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1482                                  rsc, PCMK_ACTION_STOP,
1483                                  pcmk__ar_ordered);
1484     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1485                                  rsc, PCMK_ACTION_START,
1486                                  pcmk__ar_ordered);
1487     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
1488                                  rsc, PCMK_ACTION_PROMOTE,
1489                                  pcmk__ar_ordered);
1490 
1491     // Order promote after all instances are started
1492     pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
1493                                  rsc, PCMK_ACTION_PROMOTE,
1494                                  pcmk__ar_ordered);
1495 
1496     // Order demote after all instances are demoted
1497     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
1498                                  rsc, PCMK_ACTION_DEMOTED,
1499                                  pcmk__ar_ordered);
1500 }
/* */
root/lib/pacemaker/pcmk_sched_ordering.c

DEFINITIONS
