diff --git a/cputree.c b/cputree.c index eb1981e..c195ec9 100644 --- a/cputree.c +++ b/cputree.c @@ -224,11 +224,13 @@ static struct topo_obj* add_cache_domain_to_package(struct topo_obj *cache, package->obj_type = OBJ_TYPE_PACKAGE; package->obj_type_list = &packages; package->number = packageid; + package->max_load = 0; packages = g_list_append(packages, package); } entry = g_list_find(package->children, cache); if (!entry) { + package->max_load += cache->max_load; package->children = g_list_append(package->children, cache); cache->parent = package; } @@ -264,12 +266,14 @@ static struct topo_obj* add_cpu_to_cache_domain(struct topo_obj *cpu, cache->mask = cache_mask; cache->number = cache_domain_count; cache->obj_type_list = &cache_domains; + cache->max_load = 0; cache_domains = g_list_append(cache_domains, cache); cache_domain_count++; } entry = g_list_find(cache->children, cpu); if (!entry) { + cache->max_load += cpu->max_load; cache->children = g_list_append(cache->children, cpu); cpu->parent = (struct topo_obj *)cache; } @@ -317,6 +321,8 @@ static void do_one_cpu(char *path) cpu->number = strtoul(&path[27], NULL, 10); + cpu->max_load = SLEEP_INTERVAL * 1000 * 1000 * 1000; // SLEEP_INTERVAL in nanoseconds, should be a good enough approximation + cpu_set(cpu->number, cpu_online_map); cpu_set(cpu->number, cpu->mask); @@ -398,6 +404,8 @@ static void do_one_cpu(char *path) * we override package_mask with node mask. */ node = get_numa_node(nodeid); + if (node) + node->max_load += cpu->max_load; if (node && (cpus_weight(package_mask) > cpus_weight(node->mask))) cpus_and(package_mask, package_mask, node->mask); } @@ -445,7 +453,7 @@ static void dump_balance_obj(struct topo_obj *d, void *data __attribute__((unuse log(TO_CONSOLE, LOG_INFO, "%s%s%s%sCPU number %i numa_node is ", log_indent, log_indent, log_indent, log_indent, c->number); for_each_object(cpu_numa_node(c), dump_numa_node_num, NULL); - log(TO_CONSOLE, LOG_INFO, "(load %lu)\n", (unsigned long)c->load); + log(TO_CONSOLE, LOG_INFO, "(load %lu, max %lu)\n", (unsigned long)c->load, (unsigned long)c->max_load); if (c->interrupts) for_each_irq(c->interrupts, dump_irq, (void *)18); } @@ -457,8 +465,8 @@ static void dump_cache_domain(struct topo_obj *d, void *data) log(TO_CONSOLE, LOG_INFO, "%s%sCache domain %i: numa_node is ", log_indent, log_indent, d->number); for_each_object(d->numa_nodes, dump_numa_node_num, NULL); - log(TO_CONSOLE, LOG_INFO, "cpu mask is %s (load %lu) \n", buffer, - (unsigned long)d->load); + log(TO_CONSOLE, LOG_INFO, "cpu mask is %s (load %lu, max %lu) \n", buffer, + (unsigned long)d->load, (unsigned long)d->max_load); if (d->children) for_each_object(d->children, dump_balance_obj, NULL); if (g_list_length(d->interrupts) > 0) @@ -471,8 +479,8 @@ static void dump_package(struct topo_obj *d, void *data) cpumask_scnprintf(buffer, 4096, d->mask); log(TO_CONSOLE, LOG_INFO, "Package %i: numa_node ", d->number); for_each_object(d->numa_nodes, dump_numa_node_num, NULL); - log(TO_CONSOLE, LOG_INFO, "cpu mask is %s (load %lu)\n", - buffer, (unsigned long)d->load); + log(TO_CONSOLE, LOG_INFO, "cpu mask is %s (load %lu, max %lu)\n", + buffer, (unsigned long)d->load, (unsigned long)d->max_load); if (d->children) for_each_object(d->children, dump_cache_domain, buffer); if (g_list_length(d->interrupts) > 0) diff --git a/irqlist.c b/irqlist.c index 4dd4a83..a3c9bcc 100644 --- a/irqlist.c +++ b/irqlist.c @@ -79,7 +79,7 @@ static void move_candidate_irqs(struct irq_info *info, void *data) unsigned long delta_load = 0; /* Don't rebalance irqs that don't want it */ - if (info->level == BALANCE_NONE) + if (info->level == BALANCE_NONE || info->level == BALANCE_CORE) return; /* Don't move cpus that only have one irq, regardless of load */ @@ -113,6 +113,28 @@ static void move_candidate_irqs(struct irq_info *info, void *data) info->assigned_obj = NULL; } +static void move_candidate_irqs_packing(GList *irqs, struct topo_obj *obj, struct load_balance_info *lb_info) +{ + GList *entry; + entry = g_list_last(irqs); + while (entry) { + struct irq_info *info = entry->data; + + // Only pack BALANCE_CORE irqs + if (info->level != BALANCE_CORE) + continue; + + + if (lb_info->adjustment_load >= obj->max_load / 10 * 9) { + lb_info->adjustment_load -= info->load; + log(TO_CONSOLE, LOG_INFO, "Selecting irq %d for rebalancing via packing\n", info->irq); + migrate_irq(&info->assigned_obj->interrupts, &rebalance_irq_list, info); + } + + entry = g_list_previous(entry); + } +} + static void migrate_overloaded_irqs(struct topo_obj *obj, void *data) { struct load_balance_info *info = data; @@ -143,6 +165,7 @@ static void migrate_overloaded_irqs(struct topo_obj *obj, void *data) */ info->adjustment_load = obj->load; for_each_irq(obj->interrupts, move_candidate_irqs, info); + move_candidate_irqs_packing(obj->interrupts, obj, info); } } diff --git a/numa.c b/numa.c index 13d7ebd..c710f98 100644 --- a/numa.c +++ b/numa.c @@ -58,9 +58,10 @@ static void add_one_node(int nodeid) process_one_line(path, get_mask_from_bitmap, &new->mask); } - new->obj_type = OBJ_TYPE_NODE; + new->obj_type = OBJ_TYPE_NODE; new->number = nodeid; new->obj_type_list = &numa_nodes; + new->max_load = 0; numa_nodes = g_list_append(numa_nodes, new); } diff --git a/placement.c b/placement.c index 73d87c7..2ba2cc4 100644 --- a/placement.c +++ b/placement.c @@ -76,7 +76,22 @@ static void find_best_object_packing(GList *objs, struct irq_info *info, struct entry = g_list_first(objs); while (entry) { struct topo_obj *d = entry->data; - if (d->load + info->load < 0.9e9) { + + /* + * Don't consider the unspecified numa node here + */ + if (numa_avail && (d->obj_type == OBJ_TYPE_NODE) && (d->number == NUMA_NO_NODE)) + continue; + + /* + * also don't consider any node that doesn't have at least one cpu in + * the unbanned list + */ + if ((d->obj_type == OBJ_TYPE_NODE) && + (!cpus_intersects(d->mask, unbanned_cpus))) + continue; + + if (d->load + info->load < d->max_load / 10 * 9) { place->best = d; place->best_cost = d->load + info->load; return; diff --git a/types.h b/types.h index 9693cf4..c26344c 100644 --- a/types.h +++ b/types.h @@ -46,6 +46,7 @@ enum obj_type_e { struct topo_obj { uint64_t load; uint64_t last_load; + uint64_t max_load; uint64_t irq_count; enum obj_type_e obj_type; int number;