Skip to content

Commit 1f9897b

Browse files
weizhouapacheweizhouapache
committed
extension: support Policy-Based Routing (PBR)
1 parent 24d5cdd commit 1f9897b

4 files changed

Lines changed: 407 additions & 1 deletion

File tree

‎extensions/network-namespace/README.md‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1206,6 +1206,37 @@ Built-in actions:
12061206
|--------|-------------|
12071207
| `reboot-device` | Bounces the guest veth pair (`vh-<vlan>-<id>` down → up) |
12081208
| `dump-config` | Prints namespace IP addresses, iptables rules, and per-network state to stdout |
1209+
| `pbr-create-table` | Create or update a routing-table entry in `/etc/iproute2/rt_tables` |
1210+
| `pbr-delete-table` | Remove a routing-table entry from `/etc/iproute2/rt_tables` |
1211+
| `pbr-list-tables` | List non-comment routing-table entries from `/etc/iproute2/rt_tables` |
1212+
| `pbr-add-route` | Add/replace an `ip route` entry in a specific routing table inside the namespace |
1213+
| `pbr-delete-route` | Delete an `ip route` entry from a specific routing table inside the namespace |
1214+
| `pbr-list-routes` | List routes from one table (or all tables) inside the namespace |
1215+
| `pbr-add-rule` | Add an `ip rule` policy rule mapped to a specific routing table inside the namespace |
1216+
| `pbr-delete-rule` | Delete an `ip rule` policy rule mapped to a specific routing table inside the namespace |
1217+
| `pbr-list-rules` | List policy rules (or only rules for one table) inside the namespace |
1218+
1219+
PBR action parameter keys (`--action-params` JSON):
1220+
1221+
| Action | Required keys | Optional keys |
1222+
|--------|---------------|---------------|
1223+
| `pbr-create-table` | `table-id` (or `id`), `table-name` (or `table`) | — |
1224+
| `pbr-delete-table` | `table-id` or `table-name` | — |
1225+
| `pbr-list-tables` | — | — |
1226+
| `pbr-add-route` | `table`, `route` | — |
1227+
| `pbr-delete-route` | `table`, `route` | — |
1228+
| `pbr-list-routes` | — | `table` |
1229+
| `pbr-add-rule` | `table`, `rule` | — |
1230+
| `pbr-delete-rule` | `table`, `rule` | — |
1231+
| `pbr-list-rules` | — | `table` |
1232+
1233+
Examples (equivalent to direct Linux commands):
1234+
1235+
* `{"table-id":"100","table-name":"isp1"}` → `100 isp1`
1236+
* `{"table":"isp1","route":"default via 192.168.1.1 dev eth0"}`
1237+
* `{"table":"vpn1","route":"default dev wg0"}`
1238+
* `{"table":"isp1","rule":"from 10.10.1.0/24"}`
1239+
* `{"table":"vpn1","rule":"to 10.10.2.0/24"}`
12091240

12101241
To add custom actions, place an executable script at
12111242
`${STATE_DIR}/hooks/custom-action-<name>.sh`
@@ -1306,6 +1337,34 @@ cmk runNetworkCustomAction \
13061337
"parameters[0].key=threshold" "parameters[0].value=90"
13071338
```
13081339

1340+
### PBR custom-action examples
1341+
1342+
```bash
1343+
# 1) Create action definitions (once per extension)
1344+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-create-table resourcetype=Network
1345+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-add-route resourcetype=Network
1346+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-add-rule resourcetype=Network
1347+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-list-tables resourcetype=Network
1348+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-list-routes resourcetype=Network
1349+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-list-rules resourcetype=Network
1350+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-delete-rule resourcetype=Network
1351+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-delete-route resourcetype=Network
1352+
cmk addCustomAction extensionid=<ext-uuid> name=pbr-delete-table resourcetype=Network
1353+
1354+
# 2) Execute against a network
1355+
cmk runNetworkCustomAction networkid=<network-uuid> actionid=<pbr-create-table-id> \
1356+
"parameters[0].key=table-id" "parameters[0].value=100" \
1357+
"parameters[1].key=table-name" "parameters[1].value=isp1"
1358+
1359+
cmk runNetworkCustomAction networkid=<network-uuid> actionid=<pbr-add-route-id> \
1360+
"parameters[0].key=table" "parameters[0].value=isp1" \
1361+
"parameters[1].key=route" "parameters[1].value=default via 192.168.1.1 dev eth0"
1362+
1363+
cmk runNetworkCustomAction networkid=<network-uuid> actionid=<pbr-add-rule-id> \
1364+
"parameters[0].key=table" "parameters[0].value=isp1" \
1365+
"parameters[1].key=rule" "parameters[1].value=from 10.10.1.0/24"
1366+
```
1367+
13091368
CloudStack calls `NetworkExtensionElement.runCustomAction()`, which issues:
13101369
```bash
13111370
network-namespace.sh custom-action \

‎extensions/network-namespace/network-namespace-wrapper.sh‎

Lines changed: 184 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2721,6 +2721,159 @@ PYEOF
27212721
##############################################################################
27222722
# Command: custom-action
27232723

2724+
_pbr_param() {
2725+
# Return the first non-empty key from ACTION_PARAMS_JSON.
2726+
local _k _v
2727+
for _k in "$@"; do
2728+
_v=$(_json_get "${ACTION_PARAMS_JSON}" "${_k}")
2729+
if [ -n "${_v}" ]; then
2730+
echo "${_v}"
2731+
return 0
2732+
fi
2733+
done
2734+
echo ""
2735+
}
2736+
2737+
_pbr_table_file() { echo "/etc/iproute2/rt_tables"; }
2738+
2739+
_pbr_create_table() {
2740+
local tid tname tf tmp
2741+
tid="$(_pbr_param table-id table_id id tableid)"
2742+
tname="$(_pbr_param table-name table_name name tablename table)"
2743+
[ -z "${tid}" ] && die "pbr-create-table: missing table id"
2744+
[ -z "${tname}" ] && die "pbr-create-table: missing table name"
2745+
2746+
tf="$(_pbr_table_file)"
2747+
grep -Eq "^[[:space:]]*${tid}[[:space:]]+${tname}([[:space:]]|$)" "${tf}" 2>/dev/null && {
2748+
echo "pbr-create-table: exists ${tid} ${tname}"
2749+
return 0
2750+
}
2751+
2752+
tmp=$(mktemp /tmp/cs-extnet-rt-tables-XXXXXX)
2753+
awk -v tid="${tid}" -v tname="${tname}" '
2754+
BEGIN { done = 0 }
2755+
{
2756+
if ($0 ~ "^[[:space:]]*#" || $0 ~ "^[[:space:]]*$") { print; next }
2757+
if ($1 == tid || $2 == tname) {
2758+
if (!done) {
2759+
print tid " " tname
2760+
done = 1
2761+
}
2762+
next
2763+
}
2764+
print
2765+
}
2766+
END {
2767+
if (!done) print tid " " tname
2768+
}
2769+
' "${tf}" > "${tmp}"
2770+
cat "${tmp}" > "${tf}"
2771+
rm -f "${tmp}" 2>/dev/null || true
2772+
echo "pbr-create-table: OK ${tid} ${tname}"
2773+
}
2774+
2775+
_pbr_delete_table() {
2776+
local tid tname tf tmp
2777+
tid="$(_pbr_param table-id table_id id tableid)"
2778+
tname="$(_pbr_param table-name table_name name tablename table)"
2779+
[ -z "${tid}" ] && [ -z "${tname}" ] && die "pbr-delete-table: missing table id/name"
2780+
2781+
tf="$(_pbr_table_file)"
2782+
tmp=$(mktemp /tmp/cs-extnet-rt-tables-XXXXXX)
2783+
awk -v tid="${tid}" -v tname="${tname}" '
2784+
{
2785+
if ($0 ~ "^[[:space:]]*#" || $0 ~ "^[[:space:]]*$") { print; next }
2786+
if ((tid != "" && $1 == tid) || (tname != "" && $2 == tname)) {
2787+
next
2788+
}
2789+
print
2790+
}
2791+
' "${tf}" > "${tmp}"
2792+
cat "${tmp}" > "${tf}"
2793+
rm -f "${tmp}" 2>/dev/null || true
2794+
echo "pbr-delete-table: OK id=${tid:-n/a} name=${tname:-n/a}"
2795+
}
2796+
2797+
_pbr_list_tables() {
2798+
awk '
2799+
{
2800+
if ($0 ~ "^[[:space:]]*#" || $0 ~ "^[[:space:]]*$") next
2801+
print
2802+
}
2803+
' "$(_pbr_table_file)"
2804+
}
2805+
2806+
_pbr_add_route() {
2807+
local table route
2808+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2809+
route="$(_pbr_param route route-spec route_spec)"
2810+
[ -z "${table}" ] && die "pbr-add-route: missing table"
2811+
[ -z "${route}" ] && die "pbr-add-route: missing route spec"
2812+
[ -z "${NAMESPACE}" ] && die "pbr-add-route: namespace not resolved"
2813+
2814+
# replace is idempotent and avoids duplicate route errors.
2815+
ip netns exec "${NAMESPACE}" sh -c "ip route replace ${route} table ${table}"
2816+
echo "pbr-add-route: OK table=${table} route=${route}"
2817+
}
2818+
2819+
_pbr_delete_route() {
2820+
local table route
2821+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2822+
route="$(_pbr_param route route-spec route_spec)"
2823+
[ -z "${table}" ] && die "pbr-delete-route: missing table"
2824+
[ -z "${route}" ] && die "pbr-delete-route: missing route spec"
2825+
[ -z "${NAMESPACE}" ] && die "pbr-delete-route: namespace not resolved"
2826+
2827+
ip netns exec "${NAMESPACE}" sh -c "ip route del ${route} table ${table}" 2>/dev/null || true
2828+
echo "pbr-delete-route: OK table=${table} route=${route}"
2829+
}
2830+
2831+
_pbr_list_routes() {
2832+
local table
2833+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2834+
[ -z "${NAMESPACE}" ] && die "pbr-list-routes: namespace not resolved"
2835+
if [ -n "${table}" ]; then
2836+
ip netns exec "${NAMESPACE}" ip route show table "${table}"
2837+
else
2838+
ip netns exec "${NAMESPACE}" ip route show table all
2839+
fi
2840+
}
2841+
2842+
_pbr_add_rule() {
2843+
local table rule
2844+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2845+
rule="$(_pbr_param rule rule-spec rule_spec)"
2846+
[ -z "${table}" ] && die "pbr-add-rule: missing table"
2847+
[ -z "${rule}" ] && die "pbr-add-rule: missing rule spec"
2848+
[ -z "${NAMESPACE}" ] && die "pbr-add-rule: namespace not resolved"
2849+
2850+
ip netns exec "${NAMESPACE}" sh -c "ip rule add ${rule} table ${table}" 2>/dev/null || true
2851+
echo "pbr-add-rule: OK table=${table} rule=${rule}"
2852+
}
2853+
2854+
_pbr_delete_rule() {
2855+
local table rule
2856+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2857+
rule="$(_pbr_param rule rule-spec rule_spec)"
2858+
[ -z "${table}" ] && die "pbr-delete-rule: missing table"
2859+
[ -z "${rule}" ] && die "pbr-delete-rule: missing rule spec"
2860+
[ -z "${NAMESPACE}" ] && die "pbr-delete-rule: namespace not resolved"
2861+
2862+
ip netns exec "${NAMESPACE}" sh -c "ip rule del ${rule} table ${table}" 2>/dev/null || true
2863+
echo "pbr-delete-rule: OK table=${table} rule=${rule}"
2864+
}
2865+
2866+
_pbr_list_rules() {
2867+
local table
2868+
table="$(_pbr_param table table-name table_name tablename table-id table_id id tableid)"
2869+
[ -z "${NAMESPACE}" ] && die "pbr-list-rules: namespace not resolved"
2870+
if [ -n "${table}" ]; then
2871+
ip netns exec "${NAMESPACE}" ip rule show | grep -E "[[:space:]]lookup[[:space:]]+${table}([[:space:]]|$)" || true
2872+
else
2873+
ip netns exec "${NAMESPACE}" ip rule show
2874+
fi
2875+
}
2876+
27242877
cmd_custom_action() {
27252878
NETWORK_ID=""
27262879
VPC_ID=""
@@ -2753,6 +2906,7 @@ cmd_custom_action() {
27532906
CHOSEN_ID="${VPC_ID:-${NETWORK_ID}}"
27542907

27552908
_load_state
2909+
acquire_lock "${NETWORK_ID}"
27562910

27572911
log "custom-action: network=${NETWORK_ID} ns=${NAMESPACE} action=${ACTION_NAME} params=${ACTION_PARAMS_JSON}"
27582912

@@ -2782,16 +2936,45 @@ cmd_custom_action() {
27822936
echo "=== VPC/shared state ($(_vpc_state_dir)) ==="
27832937
ls -la "$(_vpc_state_dir)/" 2>/dev/null || echo "(no vpc state)"
27842938
;;
2939+
pbr-create-table)
2940+
_pbr_create_table
2941+
;;
2942+
pbr-delete-table)
2943+
_pbr_delete_table
2944+
;;
2945+
pbr-list-tables)
2946+
_pbr_list_tables
2947+
;;
2948+
pbr-add-route)
2949+
_pbr_add_route
2950+
;;
2951+
pbr-delete-route)
2952+
_pbr_delete_route
2953+
;;
2954+
pbr-list-routes)
2955+
_pbr_list_routes
2956+
;;
2957+
pbr-add-rule)
2958+
_pbr_add_rule
2959+
;;
2960+
pbr-delete-rule)
2961+
_pbr_delete_rule
2962+
;;
2963+
pbr-list-rules)
2964+
_pbr_list_rules
2965+
;;
27852966
*)
27862967
local hook="${STATE_DIR}/hooks/custom-action-${ACTION_NAME}.sh"
27872968
if [ -x "${hook}" ]; then
27882969
exec "${hook}" --network-id "${NETWORK_ID}" --action "${ACTION_NAME}" \
27892970
--action-params "${ACTION_PARAMS_JSON}"
27902971
else
2791-
die "Unknown action '${ACTION_NAME}'. Built-ins: reboot-device, dump-config"
2972+
die "Unknown action '${ACTION_NAME}'. Built-ins: reboot-device, dump-config, pbr-*"
27922973
fi
27932974
;;
27942975
esac
2976+
2977+
release_lock
27952978
}
27962979

27972980
##############################################################################

‎framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/network/NetworkExtensionElement.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1130,6 +1130,8 @@ public String runCustomAction(Network network, String actionName, Map<String, Ob
11301130
int exitCode = process.waitFor();
11311131
String outputStr = new String(output).trim();
11321132

1133+
logger.debug("Running custom action script: {}", String.join(" ", cmdLine));
1134+
11331135
if (exitCode != 0) {
11341136
logger.error("Custom action '{}' failed (exit {}): {}", actionName, exitCode, outputStr);
11351137
return null;
@@ -2284,6 +2286,8 @@ protected void ensureExtensionDetails(Vpc vpc) {
22842286
String output = new String(process.getInputStream().readAllBytes()).trim();
22852287
int exitCode = process.waitFor();
22862288

2289+
logger.debug("Ensuring VPC network device script: {}", String.join(" ", cmdLine));
2290+
22872291
if (exitCode != 0) {
22882292
logger.warn("ensure-network-device exited {} for VPC {} — keeping current details",
22892293
exitCode, vpc.getId());

0 commit comments

Comments
 (0)