Skip to content

Commit 93531c6

Browse files
dsaherndavem330
dsahern
authored and
davem330
committed
net/ipv6: separate handling of FIB entries from dst based routes
Last step before flipping the data type for FIB entries: - use fib6_info_alloc to create FIB entries in ip6_route_info_create and addrconf_dst_alloc - use fib6_info_release in place of dst_release, ip6_rt_put and rt6_release - remove the dst_hold before calling __ip6_ins_rt or ip6_del_rt - when purging routes, drop per-cpu routes - replace inc and dec of rt6i_ref with fib6_info_hold and fib6_info_release - use rt->from since it points to the FIB entry - drop references to exception bucket, fib6_metrics and per-cpu from dst entries (those are relevant for fib entries only) Signed-off-by: David Ahern <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent a64efe1 commit 93531c6

File tree

8 files changed

+115
-152
lines changed

8 files changed

+115
-152
lines changed

‎include/net/ip6_fib.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -314,9 +314,7 @@ static inline u32 rt6_get_cookie(const struct rt6_info *rt)
314314

315315
if (rt->rt6i_flags & RTF_PCPU ||
316316
(unlikely(!list_empty(&rt->rt6i_uncached)) && rt->from))
317-
rt = rt->from;
318-
319-
rt6_get_cookie_safe(rt, &cookie);
317+
rt6_get_cookie_safe(rt->from, &cookie);
320318

321319
return cookie;
322320
}

‎include/net/ip6_route.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,7 @@ static inline int ip6_route_get_saddr(struct net *net, struct rt6_info *rt,
114114
unsigned int prefs,
115115
struct in6_addr *saddr)
116116
{
117-
struct inet6_dev *idev =
118-
rt ? ip6_dst_idev((struct dst_entry *)rt) : NULL;
117+
struct inet6_dev *idev = rt ? rt->rt6i_idev : NULL;
119118
int err = 0;
120119

121120
if (rt && rt->rt6i_prefsrc.plen)

‎net/ipv6/addrconf.c

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -916,7 +916,6 @@ void inet6_ifa_finish_destroy(struct inet6_ifaddr *ifp)
916916
pr_warn("Freeing alive inet6 address %p\n", ifp);
917917
return;
918918
}
919-
ip6_rt_put(ifp->rt);
920919

921920
kfree_rcu(ifp, rcu);
922921
}
@@ -1102,8 +1101,8 @@ ipv6_add_addr(struct inet6_dev *idev, const struct in6_addr *addr,
11021101
inet6addr_notifier_call_chain(NETDEV_UP, ifa);
11031102
out:
11041103
if (unlikely(err < 0)) {
1105-
if (rt)
1106-
ip6_rt_put(rt);
1104+
fib6_info_release(rt);
1105+
11071106
if (ifa) {
11081107
if (ifa->idev)
11091108
in6_dev_put(ifa->idev);
@@ -1191,7 +1190,7 @@ cleanup_prefix_route(struct inet6_ifaddr *ifp, unsigned long expires, bool del_r
11911190
else {
11921191
if (!(rt->rt6i_flags & RTF_EXPIRES))
11931192
fib6_set_expires(rt, expires);
1194-
ip6_rt_put(rt);
1193+
fib6_info_release(rt);
11951194
}
11961195
}
11971196
}
@@ -2375,8 +2374,7 @@ static struct rt6_info *addrconf_get_prefix_route(const struct in6_addr *pfx,
23752374
continue;
23762375
if ((rt->rt6i_flags & noflags) != 0)
23772376
continue;
2378-
if (!dst_hold_safe(&rt->dst))
2379-
rt = NULL;
2377+
fib6_info_hold(rt);
23802378
break;
23812379
}
23822380
out:
@@ -2687,7 +2685,7 @@ void addrconf_prefix_rcv(struct net_device *dev, u8 *opt, int len, bool sllao)
26872685
addrconf_prefix_route(&pinfo->prefix, pinfo->prefix_len,
26882686
dev, expires, flags, GFP_ATOMIC);
26892687
}
2690-
ip6_rt_put(rt);
2688+
fib6_info_release(rt);
26912689
}
26922690

26932691
/* Try to figure out our local address for this prefix */
@@ -3361,7 +3359,7 @@ static int fixup_permanent_addr(struct net *net,
33613359
ifp->rt = rt;
33623360
spin_unlock(&ifp->lock);
33633361

3364-
ip6_rt_put(prev);
3362+
fib6_info_release(prev);
33653363
}
33663364

33673365
if (!(ifp->flags & IFA_F_NOPREFIXROUTE)) {
@@ -5636,8 +5634,8 @@ static void __ipv6_ifa_notify(int event, struct inet6_ifaddr *ifp)
56365634
ip6_del_rt(net, rt);
56375635
}
56385636
if (ifp->rt) {
5639-
if (dst_hold_safe(&ifp->rt->dst))
5640-
ip6_del_rt(net, ifp->rt);
5637+
ip6_del_rt(net, ifp->rt);
5638+
ifp->rt = NULL;
56415639
}
56425640
rt_genid_bump_ipv6(net);
56435641
break;

‎net/ipv6/anycast.c

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ static void aca_put(struct ifacaddr6 *ac)
213213
{
214214
if (refcount_dec_and_test(&ac->aca_refcnt)) {
215215
in6_dev_put(ac->aca_idev);
216-
dst_release(&ac->aca_rt->dst);
216+
fib6_info_release(ac->aca_rt);
217217
kfree(ac);
218218
}
219219
}
@@ -231,6 +231,7 @@ static struct ifacaddr6 *aca_alloc(struct rt6_info *rt,
231231
aca->aca_addr = *addr;
232232
in6_dev_hold(idev);
233233
aca->aca_idev = idev;
234+
fib6_info_hold(rt);
234235
aca->aca_rt = rt;
235236
aca->aca_users = 1;
236237
/* aca_tstamp should be updated upon changes */
@@ -274,7 +275,7 @@ int __ipv6_dev_ac_inc(struct inet6_dev *idev, const struct in6_addr *addr)
274275
}
275276
aca = aca_alloc(rt, addr);
276277
if (!aca) {
277-
ip6_rt_put(rt);
278+
fib6_info_release(rt);
278279
err = -ENOMEM;
279280
goto out;
280281
}
@@ -330,7 +331,6 @@ int __ipv6_dev_ac_dec(struct inet6_dev *idev, const struct in6_addr *addr)
330331
write_unlock_bh(&idev->lock);
331332
addrconf_leave_solict(idev, &aca->aca_addr);
332333

333-
dst_hold(&aca->aca_rt->dst);
334334
ip6_del_rt(dev_net(idev->dev), aca->aca_rt);
335335

336336
aca_put(aca);
@@ -358,7 +358,6 @@ void ipv6_ac_destroy_dev(struct inet6_dev *idev)
358358

359359
addrconf_leave_solict(idev, &aca->aca_addr);
360360

361-
dst_hold(&aca->aca_rt->dst);
362361
ip6_del_rt(dev_net(idev->dev), aca->aca_rt);
363362

364363
aca_put(aca);

‎net/ipv6/ip6_fib.c

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ struct rt6_info *fib6_info_alloc(gfp_t gfp_flags)
170170
void fib6_info_destroy(struct rt6_info *f6i)
171171
{
172172
struct rt6_exception_bucket *bucket;
173+
struct dst_metrics *m;
173174

174175
WARN_ON(f6i->rt6i_node);
175176

@@ -201,6 +202,10 @@ void fib6_info_destroy(struct rt6_info *f6i)
201202
if (f6i->fib6_nh.nh_dev)
202203
dev_put(f6i->fib6_nh.nh_dev);
203204

205+
m = f6i->fib6_metrics;
206+
if (m != &dst_default_metrics && refcount_dec_and_test(&m->refcnt))
207+
kfree(m);
208+
204209
kfree(f6i);
205210
}
206211
EXPORT_SYMBOL_GPL(fib6_info_destroy);
@@ -714,7 +719,7 @@ static struct fib6_node *fib6_add_1(struct net *net,
714719
/* clean up an intermediate node */
715720
if (!(fn->fn_flags & RTN_RTINFO)) {
716721
RCU_INIT_POINTER(fn->leaf, NULL);
717-
rt6_release(leaf);
722+
fib6_info_release(leaf);
718723
/* remove null_entry in the root node */
719724
} else if (fn->fn_flags & RTN_TL_ROOT &&
720725
rcu_access_pointer(fn->leaf) ==
@@ -898,12 +903,32 @@ static void fib6_purge_rt(struct rt6_info *rt, struct fib6_node *fn,
898903
if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {
899904
new_leaf = fib6_find_prefix(net, table, fn);
900905
atomic_inc(&new_leaf->rt6i_ref);
906+
901907
rcu_assign_pointer(fn->leaf, new_leaf);
902-
rt6_release(rt);
908+
fib6_info_release(rt);
903909
}
904910
fn = rcu_dereference_protected(fn->parent,
905911
lockdep_is_held(&table->tb6_lock));
906912
}
913+
914+
if (rt->rt6i_pcpu) {
915+
int cpu;
916+
917+
/* release the reference to this fib entry from
918+
* all of its cached pcpu routes
919+
*/
920+
for_each_possible_cpu(cpu) {
921+
struct rt6_info **ppcpu_rt;
922+
struct rt6_info *pcpu_rt;
923+
924+
ppcpu_rt = per_cpu_ptr(rt->rt6i_pcpu, cpu);
925+
pcpu_rt = *ppcpu_rt;
926+
if (pcpu_rt) {
927+
fib6_info_release(pcpu_rt->from);
928+
pcpu_rt->from = NULL;
929+
}
930+
}
931+
}
907932
}
908933
}
909934

@@ -1099,7 +1124,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
10991124
fib6_purge_rt(iter, fn, info->nl_net);
11001125
if (rcu_access_pointer(fn->rr_ptr) == iter)
11011126
fn->rr_ptr = NULL;
1102-
rt6_release(iter);
1127+
fib6_info_release(iter);
11031128

11041129
if (nsiblings) {
11051130
/* Replacing an ECMP route, remove all siblings */
@@ -1115,7 +1140,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
11151140
fib6_purge_rt(iter, fn, info->nl_net);
11161141
if (rcu_access_pointer(fn->rr_ptr) == iter)
11171142
fn->rr_ptr = NULL;
1118-
rt6_release(iter);
1143+
fib6_info_release(iter);
11191144
nsiblings--;
11201145
info->nl_net->ipv6.rt6_stats->fib_rt_entries--;
11211146
} else {
@@ -1183,9 +1208,6 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
11831208
int replace_required = 0;
11841209
int sernum = fib6_new_sernum(info->nl_net);
11851210

1186-
if (WARN_ON_ONCE(!atomic_read(&rt->dst.__refcnt)))
1187-
return -EINVAL;
1188-
11891211
if (info->nlh) {
11901212
if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
11911213
allow_create = 0;
@@ -1300,7 +1322,7 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
13001322
if (pn_leaf == rt) {
13011323
pn_leaf = NULL;
13021324
RCU_INIT_POINTER(pn->leaf, NULL);
1303-
atomic_dec(&rt->rt6i_ref);
1325+
fib6_info_release(rt);
13041326
}
13051327
if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) {
13061328
pn_leaf = fib6_find_prefix(info->nl_net, table,
@@ -1312,7 +1334,7 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
13121334
info->nl_net->ipv6.fib6_null_entry;
13131335
}
13141336
#endif
1315-
atomic_inc(&pn_leaf->rt6i_ref);
1337+
fib6_info_hold(pn_leaf);
13161338
rcu_assign_pointer(pn->leaf, pn_leaf);
13171339
}
13181340
}
@@ -1334,10 +1356,6 @@ int fib6_add(struct fib6_node *root, struct rt6_info *rt,
13341356
(fn->fn_flags & RTN_TL_ROOT &&
13351357
!rcu_access_pointer(fn->leaf))))
13361358
fib6_repair_tree(info->nl_net, table, fn);
1337-
/* Always release dst as dst->__refcnt is guaranteed
1338-
* to be taken before entering this function
1339-
*/
1340-
dst_release_immediate(&rt->dst);
13411359
return err;
13421360
}
13431361

@@ -1637,7 +1655,7 @@ static struct fib6_node *fib6_repair_tree(struct net *net,
16371655
new_fn_leaf = net->ipv6.fib6_null_entry;
16381656
}
16391657
#endif
1640-
atomic_inc(&new_fn_leaf->rt6i_ref);
1658+
fib6_info_hold(new_fn_leaf);
16411659
rcu_assign_pointer(fn->leaf, new_fn_leaf);
16421660
return pn;
16431661
}
@@ -1693,7 +1711,7 @@ static struct fib6_node *fib6_repair_tree(struct net *net,
16931711
return pn;
16941712

16951713
RCU_INIT_POINTER(pn->leaf, NULL);
1696-
rt6_release(pn_leaf);
1714+
fib6_info_release(pn_leaf);
16971715
fn = pn;
16981716
}
16991717
}
@@ -1763,7 +1781,7 @@ static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
17631781
call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt, NULL);
17641782
if (!info->skip_notify)
17651783
inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
1766-
rt6_release(rt);
1784+
fib6_info_release(rt);
17671785
}
17681786

17691787
/* Need to own table->tb6_lock */
@@ -2261,9 +2279,8 @@ static int ipv6_route_seq_show(struct seq_file *seq, void *v)
22612279

22622280
dev = rt->fib6_nh.nh_dev;
22632281
seq_printf(seq, " %08x %08x %08x %08x %8s\n",
2264-
rt->rt6i_metric, atomic_read(&rt->dst.__refcnt),
2265-
rt->dst.__use, rt->rt6i_flags,
2266-
dev ? dev->name : "");
2282+
rt->rt6i_metric, atomic_read(&rt->rt6i_ref), 0,
2283+
rt->rt6i_flags, dev ? dev->name : "");
22672284
iter->w.leaf = NULL;
22682285
return 0;
22692286
}

‎net/ipv6/ip6_output.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -968,7 +968,8 @@ static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
968968
if (!had_dst)
969969
*dst = ip6_route_output(net, sk, fl6);
970970
rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
971-
err = ip6_route_get_saddr(net, rt, &fl6->daddr,
971+
err = ip6_route_get_saddr(net, rt ? rt->from : NULL,
972+
&fl6->daddr,
972973
sk ? inet6_sk(sk)->srcprefs : 0,
973974
&fl6->saddr);
974975
if (err)

‎net/ipv6/ndisc.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,7 +1283,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
12831283
ND_PRINTK(0, err,
12841284
"RA: %s got default router without neighbour\n",
12851285
__func__);
1286-
ip6_rt_put(rt);
1286+
fib6_info_release(rt);
12871287
return;
12881288
}
12891289
}
@@ -1313,7 +1313,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
13131313
ND_PRINTK(0, err,
13141314
"RA: %s got default router without neighbour\n",
13151315
__func__);
1316-
ip6_rt_put(rt);
1316+
fib6_info_release(rt);
13171317
return;
13181318
}
13191319
neigh->flags |= NTF_ROUTER;
@@ -1499,7 +1499,7 @@ static void ndisc_router_discovery(struct sk_buff *skb)
14991499
ND_PRINTK(2, warn, "RA: invalid RA options\n");
15001500
}
15011501
out:
1502-
ip6_rt_put(rt);
1502+
fib6_info_release(rt);
15031503
if (neigh)
15041504
neigh_release(neigh);
15051505
}

0 commit comments

Comments
 (0)