pktgen.c 90.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8
/*
 * Authors:
 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se>
 *                             Uppsala University and
 *                             Swedish University of Agricultural Sciences
 *
 * Alexey Kuznetsov  <kuznet@ms2.inr.ac.ru>
 * Ben Greear <greearb@candelatech.com>
9
 * Jens Låås <jens.laas@data.slu.se>
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 *
 * A tool for loading the network with preconfigurated packets.
18
 * The tool is implemented as a linux module.  Parameters are output
L
Linus Torvalds 已提交
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
 * device, delay (to hard_xmit), number of packets, and whether
 * to use multiple SKBs or just the same one.
 * pktgen uses the installed interface's output routine.
 *
 * Additional hacking by:
 *
 * Jens.Laas@data.slu.se
 * Improved by ANK. 010120.
 * Improved by ANK even more. 010212.
 * MAC address typo fixed. 010417 --ro
 * Integrated.  020301 --DaveM
 * Added multiskb option 020301 --DaveM
 * Scaling of results. 020417--sigurdur@linpro.no
 * Significant re-work of the module:
 *   *  Convert to threaded model to more efficiently be able to transmit
 *       and receive on multiple interfaces at once.
 *   *  Converted many counters to __u64 to allow longer runs.
 *   *  Allow configuration of ranges, like min/max IP address, MACs,
 *       and UDP-ports, for both source and destination, and can
 *       set to use a random distribution or sequentially walk the range.
 *   *  Can now change most values after starting.
 *   *  Place 12-byte packet in UDP payload with magic number,
 *       sequence number, and timestamp.
 *   *  Add receiver code that detects dropped pkts, re-ordered pkts, and
 *       latencies (with micro-second) precision.
 *   *  Add IOCTL interface to easily get counters & configuration.
 *   --Ben Greear <greearb@candelatech.com>
 *
47 48
 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct
 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0
L
Linus Torvalds 已提交
49
 * as a "fastpath" with a configurable number of clones after alloc's.
50 51
 * clone_skb=0 means all packets are allocated this also means ranges time
 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100
L
Linus Torvalds 已提交
52 53
 * clones.
 *
54
 * Also moved to /proc/net/pktgen/
L
Linus Torvalds 已提交
55 56 57 58 59 60 61 62
 * --ro
 *
 * Sept 10:  Fixed threading/locking.  Lots of bone-headed and more clever
 *    mistakes.  Also merged in DaveM's patch in the -pre6 patch.
 * --Ben Greear <greearb@candelatech.com>
 *
 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br)
 *
63
 *
L
Linus Torvalds 已提交
64 65 66 67
 * 021124 Finished major redesign and rewrite for new functionality.
 * See Documentation/networking/pktgen.txt for how to use this.
 *
 * The new operation:
68 69 70 71
 * For each CPU one thread/process is created at start. This process checks
 * for running devices in the if_list and sends packets until count is 0 it
 * also the thread checks the thread->control which is used for inter-process
 * communication. controlling process "posts" operations to the threads this
L
Linus Torvalds 已提交
72 73 74
 * way. The if_lock should be possible to remove when add/rem_device is merged
 * into this too.
 *
75 76
 * By design there should only be *one* "controlling" process. In practice
 * multiple write accesses gives unpredictable result. Understood by "write"
L
Linus Torvalds 已提交
77
 * to /proc gives result code thats should be read be the "writer".
78
 * For practical use this should be no problem.
L
Linus Torvalds 已提交
79
 *
80 81
 * Note when adding devices to a specific CPU there good idea to also assign
 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU.
L
Linus Torvalds 已提交
82 83
 * --ro
 *
84
 * Fix refcount off by one if first packet fails, potential null deref,
L
Linus Torvalds 已提交
85 86 87 88 89 90 91
 * memleak 030710- KJP
 *
 * First "ranges" functionality for ipv6 030726 --ro
 *
 * Included flow support. 030802 ANK.
 *
 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org>
92
 *
L
Linus Torvalds 已提交
93 94 95
 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419
 * ia64 compilation fix from  Aron Griffis <aron@hp.com> 040604
 *
96
 * New xmit() return, do_div and misc clean up by Stephen Hemminger
L
Linus Torvalds 已提交
97 98
 * <shemminger@osdl.org> 040923
 *
99
 * Randy Dunlap fixed u64 printk compiler waring
L
Linus Torvalds 已提交
100 101 102 103
 *
 * Remove FCS from BW calculation.  Lennert Buytenhek <buytenh@wantstofly.org>
 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213
 *
104
 * Corrections from Nikolai Malykh (nmalykh@bilim.com)
L
Linus Torvalds 已提交
105 106
 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230
 *
107
 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com>
L
Linus Torvalds 已提交
108
 * 050103
109 110 111
 *
 * MPLS support by Steven Whitehouse <steve@chygwyn.com>
 *
F
Francesco Fondelli 已提交
112 113
 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com>
 *
A
Adit Ranadive 已提交
114 115 116
 * Fixed src_mac command to set source mac of packet to value specified in
 * command by Adit Ranadive <adit.262@gmail.com>
 *
L
Linus Torvalds 已提交
117
 */
J
Joe Perches 已提交
118 119 120

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

L
Linus Torvalds 已提交
121 122 123 124 125
#include <linux/sys.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/kernel.h>
126
#include <linux/mutex.h>
L
Linus Torvalds 已提交
127 128 129 130 131 132 133 134 135
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/unistd.h>
#include <linux/string.h>
#include <linux/ptrace.h>
#include <linux/errno.h>
#include <linux/ioport.h>
#include <linux/interrupt.h>
136
#include <linux/capability.h>
S
Stephen Hemminger 已提交
137
#include <linux/hrtimer.h>
A
Andrew Morton 已提交
138
#include <linux/freezer.h>
L
Linus Torvalds 已提交
139 140
#include <linux/delay.h>
#include <linux/timer.h>
141
#include <linux/list.h>
L
Linus Torvalds 已提交
142 143 144 145 146 147 148
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/inet.h>
#include <linux/inetdevice.h>
#include <linux/rtnetlink.h>
#include <linux/if_arp.h>
F
Francesco Fondelli 已提交
149
#include <linux/if_vlan.h>
L
Linus Torvalds 已提交
150 151 152 153 154
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/udp.h>
#include <linux/proc_fs.h>
155
#include <linux/seq_file.h>
L
Linus Torvalds 已提交
156
#include <linux/wait.h>
157
#include <linux/etherdevice.h>
158
#include <linux/kthread.h>
159
#include <linux/prefetch.h>
160
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
161 162 163
#include <net/checksum.h>
#include <net/ipv6.h>
#include <net/addrconf.h>
J
Jamal Hadi Salim 已提交
164 165 166
#ifdef CONFIG_XFRM
#include <net/xfrm.h>
#endif
L
Linus Torvalds 已提交
167 168
#include <asm/byteorder.h>
#include <linux/rcupdate.h>
J
Jiri Slaby 已提交
169
#include <linux/bitops.h>
170 171 172
#include <linux/io.h>
#include <linux/timex.h>
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
173
#include <asm/dma.h>
L
Luiz Capitulino 已提交
174
#include <asm/div64.h>		/* do_div */
L
Linus Torvalds 已提交
175

176
#define VERSION	"2.74"
L
Linus Torvalds 已提交
177
#define IP_NAME_SZ 32
178
#define MAX_MPLS_LABELS 16 /* This is the max label stack depth */
179
#define MPLS_STACK_BOTTOM htonl(0x00000100)
L
Linus Torvalds 已提交
180

J
Joe Perches 已提交
181 182
#define func_enter() pr_debug("entering %s\n", __func__);

L
Linus Torvalds 已提交
183
/* Device flag bits */
L
Luiz Capitulino 已提交
184 185 186 187 188 189 190 191
#define F_IPSRC_RND   (1<<0)	/* IP-Src Random  */
#define F_IPDST_RND   (1<<1)	/* IP-Dst Random  */
#define F_UDPSRC_RND  (1<<2)	/* UDP-Src Random */
#define F_UDPDST_RND  (1<<3)	/* UDP-Dst Random */
#define F_MACSRC_RND  (1<<4)	/* MAC-Src Random */
#define F_MACDST_RND  (1<<5)	/* MAC-Dst Random */
#define F_TXSIZE_RND  (1<<6)	/* Transmit size is random */
#define F_IPV6        (1<<7)	/* Interface in IPV6 Mode */
192
#define F_MPLS_RND    (1<<8)	/* Random MPLS labels */
F
Francesco Fondelli 已提交
193 194
#define F_VID_RND     (1<<9)	/* Random VLAN ID */
#define F_SVID_RND    (1<<10)	/* Random SVLAN ID */
195
#define F_FLOW_SEQ    (1<<11)	/* Sequential flows */
J
Jamal Hadi Salim 已提交
196
#define F_IPSEC_ON    (1<<12)	/* ipsec on for flows */
R
Robert Olsson 已提交
197
#define F_QUEUE_MAP_RND (1<<13)	/* queue map Random */
R
Robert Olsson 已提交
198
#define F_QUEUE_MAP_CPU (1<<14)	/* queue map mirrors smp_processor_id() */
R
Robert Olsson 已提交
199
#define F_NODE          (1<<15)	/* Node memory alloc*/
L
Linus Torvalds 已提交
200 201

/* Thread control flag bits */
202 203 204 205
#define T_STOP        (1<<0)	/* Stop run */
#define T_RUN         (1<<1)	/* Start run */
#define T_REMDEVALL   (1<<2)	/* Remove all devs */
#define T_REMDEV      (1<<3)	/* Remove one dev */
L
Linus Torvalds 已提交
206 207 208 209 210 211 212

/* If lock -- can be removed after some work */
#define   if_lock(t)           spin_lock(&(t->if_lock));
#define   if_unlock(t)           spin_unlock(&(t->if_lock));

/* Used to help with determining the pkts on receive */
#define PKTGEN_MAGIC 0xbe9be955
213 214
#define PG_PROC_DIR "pktgen"
#define PGCTRL	    "pgctrl"
215
static struct proc_dir_entry *pg_proc_dir;
L
Linus Torvalds 已提交
216 217 218

#define MAX_CFLOWS  65536

F
Francesco Fondelli 已提交
219 220 221
#define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4)
#define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4)

L
Luiz Capitulino 已提交
222
struct flow_state {
A
Al Viro 已提交
223
	__be32 cur_daddr;
L
Luiz Capitulino 已提交
224
	int count;
J
Jamal Hadi Salim 已提交
225 226 227
#ifdef CONFIG_XFRM
	struct xfrm_state *x;
#endif
228
	__u32 flags;
L
Linus Torvalds 已提交
229 230
};

231 232 233
/* flow flag bits */
#define F_INIT   (1<<0)		/* flow has been initialized */

L
Linus Torvalds 已提交
234 235 236 237
struct pktgen_dev {
	/*
	 * Try to keep frequent/infrequent used vars. separated.
	 */
238 239
	struct proc_dir_entry *entry;	/* proc file */
	struct pktgen_thread *pg_thread;/* the owner */
240
	struct list_head list;		/* chaining in the thread's run-queue */
L
Linus Torvalds 已提交
241

242
	int running;		/* if false, the test will stop */
L
Linus Torvalds 已提交
243

L
Luiz Capitulino 已提交
244 245 246 247
	/* If min != max, then we will either do a linear iteration, or
	 * we will do a random selection from within the range.
	 */
	__u32 flags;
248 249
	int removal_mark;	/* non-zero => the device is marked for
				 * removal by worker thread */
L
Linus Torvalds 已提交
250

L
Luiz Capitulino 已提交
251 252
	int min_pkt_size;	/* = ETH_ZLEN; */
	int max_pkt_size;	/* = ETH_ZLEN; */
253
	int pkt_overhead;	/* overhead for MPLS, VLANs, IPSEC etc */
L
Luiz Capitulino 已提交
254
	int nfrags;
E
Eric Dumazet 已提交
255
	struct page *page;
256 257
	u64 delay;		/* nano-seconds */

L
Luiz Capitulino 已提交
258 259 260
	__u64 count;		/* Default No packets to send */
	__u64 sofar;		/* How many pkts we've sent so far */
	__u64 tx_bytes;		/* How many bytes we've transmitted */
261
	__u64 errors;		/* Errors when trying to transmit, */
L
Luiz Capitulino 已提交
262 263 264 265 266 267

	/* runtime counters relating to clone_skb */

	__u64 allocated_skbs;
	__u32 clone_count;
	int last_ok;		/* Was last skb sent?
268 269
				 * Or a failed transmit of some sort?
				 * This will keep sequence numbers in order
L
Luiz Capitulino 已提交
270
				 */
271 272 273 274 275
	ktime_t next_tx;
	ktime_t started_at;
	ktime_t stopped_at;
	u64	idle_acc;	/* nano-seconds */

L
Luiz Capitulino 已提交
276 277
	__u32 seq_num;

278 279 280 281 282 283 284 285
	int clone_skb;		/*
				 * Use multiple SKBs during packet gen.
				 * If this number is greater than 1, then
				 * that many copies of the same packet will be
				 * sent before a new packet is allocated.
				 * If you want to send 1024 identical packets
				 * before creating a new packet,
				 * set clone_skb to 1024.
L
Luiz Capitulino 已提交
286 287 288 289 290 291 292 293 294 295 296
				 */

	char dst_min[IP_NAME_SZ];	/* IP, ie 1.2.3.4 */
	char dst_max[IP_NAME_SZ];	/* IP, ie 1.2.3.4 */
	char src_min[IP_NAME_SZ];	/* IP, ie 1.2.3.4 */
	char src_max[IP_NAME_SZ];	/* IP, ie 1.2.3.4 */

	struct in6_addr in6_saddr;
	struct in6_addr in6_daddr;
	struct in6_addr cur_in6_daddr;
	struct in6_addr cur_in6_saddr;
L
Linus Torvalds 已提交
297
	/* For ranges */
L
Luiz Capitulino 已提交
298 299 300 301 302 303 304 305
	struct in6_addr min_in6_daddr;
	struct in6_addr max_in6_daddr;
	struct in6_addr min_in6_saddr;
	struct in6_addr max_in6_saddr;

	/* If we're doing ranges, random or incremental, then this
	 * defines the min/max for those ranges.
	 */
A
Al Viro 已提交
306 307 308 309
	__be32 saddr_min;	/* inclusive, source IP address */
	__be32 saddr_max;	/* exclusive, source IP address */
	__be32 daddr_min;	/* inclusive, dest IP address */
	__be32 daddr_max;	/* exclusive, dest IP address */
L
Luiz Capitulino 已提交
310 311 312 313 314 315

	__u16 udp_src_min;	/* inclusive, source UDP port */
	__u16 udp_src_max;	/* exclusive, source UDP port */
	__u16 udp_dst_min;	/* inclusive, dest UDP port */
	__u16 udp_dst_max;	/* exclusive, dest UDP port */

F
Francesco Fondelli 已提交
316
	/* DSCP + ECN */
317 318 319 320
	__u8 tos;            /* six MSB of (former) IPv4 TOS
				are for dscp codepoint */
	__u8 traffic_class;  /* ditto for the (former) Traffic Class in IPv6
				(see RFC 3260, sec. 4) */
F
Francesco Fondelli 已提交
321

322 323 324 325
	/* MPLS */
	unsigned nr_labels;	/* Depth of stack, 0 = no MPLS */
	__be32 labels[MAX_MPLS_LABELS];

F
Francesco Fondelli 已提交
326 327 328 329 330 331 332 333 334
	/* VLAN/SVLAN (802.1Q/Q-in-Q) */
	__u8  vlan_p;
	__u8  vlan_cfi;
	__u16 vlan_id;  /* 0xffff means no vlan tag */

	__u8  svlan_p;
	__u8  svlan_cfi;
	__u16 svlan_id; /* 0xffff means no svlan tag */

L
Luiz Capitulino 已提交
335 336 337 338 339 340 341 342
	__u32 src_mac_count;	/* How many MACs to iterate through */
	__u32 dst_mac_count;	/* How many MACs to iterate through */

	unsigned char dst_mac[ETH_ALEN];
	unsigned char src_mac[ETH_ALEN];

	__u32 cur_dst_mac_offset;
	__u32 cur_src_mac_offset;
A
Al Viro 已提交
343 344
	__be32 cur_saddr;
	__be32 cur_daddr;
E
Eric Dumazet 已提交
345
	__u16 ip_id;
L
Luiz Capitulino 已提交
346 347
	__u16 cur_udp_dst;
	__u16 cur_udp_src;
R
Robert Olsson 已提交
348
	__u16 cur_queue_map;
L
Luiz Capitulino 已提交
349
	__u32 cur_pkt_size;
350
	__u32 last_pkt_size;
L
Luiz Capitulino 已提交
351 352 353 354 355 356 357 358 359 360 361 362

	__u8 hh[14];
	/* = {
	   0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB,

	   We fill in SRC address later
	   0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	   0x08, 0x00
	   };
	 */
	__u16 pad;		/* pad out the hh struct to an even 16 bytes */

363
	struct sk_buff *skb;	/* skb we are to transmit next, used for when we
L
Luiz Capitulino 已提交
364 365
				 * are transmitting the same one multiple times
				 */
366 367 368 369 370 371 372 373
	struct net_device *odev; /* The out-going device.
				  * Note that the device should have it's
				  * pg_info pointer pointing back to this
				  * device.
				  * Set when the user specifies the out-going
				  * device name (not when the inject is
				  * started as it used to do.)
				  */
E
Eric Dumazet 已提交
374
	char odevname[32];
L
Linus Torvalds 已提交
375
	struct flow_state *flows;
L
Luiz Capitulino 已提交
376 377 378
	unsigned cflows;	/* Concurrent flows (config) */
	unsigned lflow;		/* Flow length  (config) */
	unsigned nflows;	/* accumulated flows (stats) */
379
	unsigned curfl;		/* current sequenced flow (state)*/
R
Robert Olsson 已提交
380 381 382

	u16 queue_map_min;
	u16 queue_map_max;
J
John Fastabend 已提交
383
	__u32 skb_priority;	/* skb priority field */
R
Robert Olsson 已提交
384
	int node;               /* Memory node */
R
Robert Olsson 已提交
385

J
Jamal Hadi Salim 已提交
386 387 388 389
#ifdef CONFIG_XFRM
	__u8	ipsmode;		/* IPSEC mode (config) */
	__u8	ipsproto;		/* IPSEC type (config) */
#endif
390
	char result[512];
L
Linus Torvalds 已提交
391 392 393
};

struct pktgen_hdr {
A
Al Viro 已提交
394 395 396 397
	__be32 pgh_magic;
	__be32 seq_num;
	__be32 tv_sec;
	__be32 tv_usec;
L
Linus Torvalds 已提交
398 399
};

E
Eric Dumazet 已提交
400 401
static bool pktgen_exiting __read_mostly;

L
Linus Torvalds 已提交
402
struct pktgen_thread {
403
	spinlock_t if_lock;		/* for list of devices */
404
	struct list_head if_list;	/* All device here */
405
	struct list_head th_list;
406
	struct task_struct *tsk;
L
Luiz Capitulino 已提交
407 408
	char result[512];

409 410
	/* Field for thread to receive "posted" events terminate,
	   stop ifs etc. */
L
Luiz Capitulino 已提交
411 412

	u32 control;
L
Linus Torvalds 已提交
413 414
	int cpu;

L
Luiz Capitulino 已提交
415
	wait_queue_head_t queue;
416
	struct completion start_done;
L
Linus Torvalds 已提交
417 418 419 420 421
};

#define REMOVE 1
#define FIND   0

422
static inline ktime_t ktime_now(void)
L
Linus Torvalds 已提交
423
{
424 425 426 427
	struct timespec ts;
	ktime_get_ts(&ts);

	return timespec_to_ktime(ts);
L
Linus Torvalds 已提交
428 429
}

430 431
/* This works even if 32 bit because of careful byte order choice */
static inline int ktime_lt(const ktime_t cmp1, const ktime_t cmp2)
L
Linus Torvalds 已提交
432
{
433
	return cmp1.tv64 < cmp2.tv64;
L
Linus Torvalds 已提交
434 435
}

S
Stephen Hemminger 已提交
436
static const char version[] =
J
Joe Perches 已提交
437 438
	"Packet Generator for packet performance testing. "
	"Version: " VERSION "\n";
L
Linus Torvalds 已提交
439

L
Luiz Capitulino 已提交
440 441 442
static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i);
static int pktgen_add_device(struct pktgen_thread *t, const char *ifname);
static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
E
Eric Dumazet 已提交
443
					  const char *ifname, bool exact);
L
Linus Torvalds 已提交
444 445
static int pktgen_device_event(struct notifier_block *, unsigned long, void *);
static void pktgen_run_all_threads(void);
446
static void pktgen_reset_all_threads(void);
L
Linus Torvalds 已提交
447
static void pktgen_stop_all_threads_ifs(void);
S
Stephen Hemminger 已提交
448

L
Luiz Capitulino 已提交
449
static void pktgen_stop(struct pktgen_thread *t);
L
Linus Torvalds 已提交
450
static void pktgen_clear_counters(struct pktgen_dev *pkt_dev);
451

L
Luiz Capitulino 已提交
452
static unsigned int scan_ip6(const char *s, char ip[16]);
L
Linus Torvalds 已提交
453 454

/* Module parameters, defaults. */
455 456 457 458
static int pg_count_d __read_mostly = 1000;
static int pg_delay_d __read_mostly;
static int pg_clone_skb_d  __read_mostly;
static int debug  __read_mostly;
L
Linus Torvalds 已提交
459

460
static DEFINE_MUTEX(pktgen_thread_lock);
461
static LIST_HEAD(pktgen_threads);
L
Linus Torvalds 已提交
462 463 464 465 466 467

static struct notifier_block pktgen_notifier_block = {
	.notifier_call = pktgen_device_event,
};

/*
468
 * /proc handling functions
L
Linus Torvalds 已提交
469 470 471
 *
 */

472
static int pgctrl_show(struct seq_file *seq, void *v)
L
Luiz Capitulino 已提交
473
{
S
Stephen Hemminger 已提交
474
	seq_puts(seq, version);
475
	return 0;
L
Linus Torvalds 已提交
476 477
}

478 479
static ssize_t pgctrl_write(struct file *file, const char __user *buf,
			    size_t count, loff_t *ppos)
L
Linus Torvalds 已提交
480 481
{
	int err = 0;
482
	char data[128];
L
Linus Torvalds 已提交
483

L
Luiz Capitulino 已提交
484 485
	if (!capable(CAP_NET_ADMIN)) {
		err = -EPERM;
L
Linus Torvalds 已提交
486
		goto out;
L
Luiz Capitulino 已提交
487
	}
L
Linus Torvalds 已提交
488

489 490
	if (count > sizeof(data))
		count = sizeof(data);
L
Linus Torvalds 已提交
491 492

	if (copy_from_user(data, buf, count)) {
493
		err = -EFAULT;
494
		goto out;
L
Luiz Capitulino 已提交
495 496
	}
	data[count - 1] = 0;	/* Make string */
L
Linus Torvalds 已提交
497

L
Luiz Capitulino 已提交
498
	if (!strcmp(data, "stop"))
L
Linus Torvalds 已提交
499 500
		pktgen_stop_all_threads_ifs();

L
Luiz Capitulino 已提交
501
	else if (!strcmp(data, "start"))
L
Linus Torvalds 已提交
502 503
		pktgen_run_all_threads();

504 505 506
	else if (!strcmp(data, "reset"))
		pktgen_reset_all_threads();

L
Luiz Capitulino 已提交
507
	else
J
Joe Perches 已提交
508
		pr_warning("Unknown command: %s\n", data);
L
Linus Torvalds 已提交
509 510 511

	err = count;

L
Luiz Capitulino 已提交
512 513
out:
	return err;
L
Linus Torvalds 已提交
514 515
}

516 517 518 519 520
static int pgctrl_open(struct inode *inode, struct file *file)
{
	return single_open(file, pgctrl_show, PDE(inode)->data);
}

521
static const struct file_operations pktgen_fops = {
L
Luiz Capitulino 已提交
522 523 524 525 526 527
	.owner   = THIS_MODULE,
	.open    = pgctrl_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.write   = pgctrl_write,
	.release = single_release,
528 529 530
};

static int pktgen_if_show(struct seq_file *seq, void *v)
L
Linus Torvalds 已提交
531
{
S
Stephen Hemminger 已提交
532
	const struct pktgen_dev *pkt_dev = seq->private;
533 534
	ktime_t stopped;
	u64 idle;
L
Luiz Capitulino 已提交
535 536 537 538 539 540 541

	seq_printf(seq,
		   "Params: count %llu  min_pkt_size: %u  max_pkt_size: %u\n",
		   (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size,
		   pkt_dev->max_pkt_size);

	seq_printf(seq,
542 543
		   "     frags: %d  delay: %llu  clone_skb: %d  ifname: %s\n",
		   pkt_dev->nfrags, (unsigned long long) pkt_dev->delay,
E
Eric Dumazet 已提交
544
		   pkt_dev->clone_skb, pkt_dev->odevname);
L
Luiz Capitulino 已提交
545 546 547 548

	seq_printf(seq, "     flows: %u flowlen: %u\n", pkt_dev->cflows,
		   pkt_dev->lflow);

R
Robert Olsson 已提交
549 550 551 552 553
	seq_printf(seq,
		   "     queue_map_min: %u  queue_map_max: %u\n",
		   pkt_dev->queue_map_min,
		   pkt_dev->queue_map_max);

J
John Fastabend 已提交
554 555 556 557
	if (pkt_dev->skb_priority)
		seq_printf(seq, "     skb_priority: %u\n",
			   pkt_dev->skb_priority);

L
Luiz Capitulino 已提交
558 559
	if (pkt_dev->flags & F_IPV6) {
		seq_printf(seq,
560 561 562 563 564 565
			   "     saddr: %pI6c  min_saddr: %pI6c  max_saddr: %pI6c\n"
			   "     daddr: %pI6c  min_daddr: %pI6c  max_daddr: %pI6c\n",
			   &pkt_dev->in6_saddr,
			   &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr,
			   &pkt_dev->in6_daddr,
			   &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr);
566 567 568 569
	} else {
		seq_printf(seq,
			   "     dst_min: %s  dst_max: %s\n",
			   pkt_dev->dst_min, pkt_dev->dst_max);
L
Luiz Capitulino 已提交
570
		seq_printf(seq,
571 572 573
			   "        src_min: %s  src_max: %s\n",
			   pkt_dev->src_min, pkt_dev->src_max);
	}
L
Linus Torvalds 已提交
574

L
Luiz Capitulino 已提交
575
	seq_puts(seq, "     src_mac: ");
L
Linus Torvalds 已提交
576

J
Johannes Berg 已提交
577 578 579
	seq_printf(seq, "%pM ",
		   is_zero_ether_addr(pkt_dev->src_mac) ?
			     pkt_dev->odev->dev_addr : pkt_dev->src_mac);
L
Luiz Capitulino 已提交
580 581

	seq_printf(seq, "dst_mac: ");
J
Johannes Berg 已提交
582
	seq_printf(seq, "%pM\n", pkt_dev->dst_mac);
L
Luiz Capitulino 已提交
583 584

	seq_printf(seq,
585 586
		   "     udp_src_min: %d  udp_src_max: %d"
		   "  udp_dst_min: %d  udp_dst_max: %d\n",
L
Luiz Capitulino 已提交
587 588 589 590
		   pkt_dev->udp_src_min, pkt_dev->udp_src_max,
		   pkt_dev->udp_dst_min, pkt_dev->udp_dst_max);

	seq_printf(seq,
591
		   "     src_mac_count: %d  dst_mac_count: %d\n",
L
Luiz Capitulino 已提交
592
		   pkt_dev->src_mac_count, pkt_dev->dst_mac_count);
L
Linus Torvalds 已提交
593

594 595 596
	if (pkt_dev->nr_labels) {
		unsigned i;
		seq_printf(seq, "     mpls: ");
597
		for (i = 0; i < pkt_dev->nr_labels; i++)
598 599 600 601
			seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]),
				   i == pkt_dev->nr_labels-1 ? "\n" : ", ");
	}

602
	if (pkt_dev->vlan_id != 0xffff)
F
Francesco Fondelli 已提交
603
		seq_printf(seq, "     vlan_id: %u  vlan_p: %u  vlan_cfi: %u\n",
604 605
			   pkt_dev->vlan_id, pkt_dev->vlan_p,
			   pkt_dev->vlan_cfi);
F
Francesco Fondelli 已提交
606

607
	if (pkt_dev->svlan_id != 0xffff)
F
Francesco Fondelli 已提交
608
		seq_printf(seq, "     svlan_id: %u  vlan_p: %u  vlan_cfi: %u\n",
609 610
			   pkt_dev->svlan_id, pkt_dev->svlan_p,
			   pkt_dev->svlan_cfi);
F
Francesco Fondelli 已提交
611

612
	if (pkt_dev->tos)
F
Francesco Fondelli 已提交
613 614
		seq_printf(seq, "     tos: 0x%02x\n", pkt_dev->tos);

615
	if (pkt_dev->traffic_class)
F
Francesco Fondelli 已提交
616 617
		seq_printf(seq, "     traffic_class: 0x%02x\n", pkt_dev->traffic_class);

R
Robert Olsson 已提交
618 619 620
	if (pkt_dev->node >= 0)
		seq_printf(seq, "     node: %d\n", pkt_dev->node);

621 622
	seq_printf(seq, "     Flags: ");

L
Luiz Capitulino 已提交
623 624
	if (pkt_dev->flags & F_IPV6)
		seq_printf(seq, "IPV6  ");
L
Linus Torvalds 已提交
625

L
Luiz Capitulino 已提交
626 627
	if (pkt_dev->flags & F_IPSRC_RND)
		seq_printf(seq, "IPSRC_RND  ");
L
Linus Torvalds 已提交
628

L
Luiz Capitulino 已提交
629 630
	if (pkt_dev->flags & F_IPDST_RND)
		seq_printf(seq, "IPDST_RND  ");
L
Linus Torvalds 已提交
631

L
Luiz Capitulino 已提交
632 633
	if (pkt_dev->flags & F_TXSIZE_RND)
		seq_printf(seq, "TXSIZE_RND  ");
L
Linus Torvalds 已提交
634

L
Luiz Capitulino 已提交
635 636
	if (pkt_dev->flags & F_UDPSRC_RND)
		seq_printf(seq, "UDPSRC_RND  ");
L
Linus Torvalds 已提交
637

L
Luiz Capitulino 已提交
638 639
	if (pkt_dev->flags & F_UDPDST_RND)
		seq_printf(seq, "UDPDST_RND  ");
L
Linus Torvalds 已提交
640

641 642 643
	if (pkt_dev->flags & F_MPLS_RND)
		seq_printf(seq,  "MPLS_RND  ");

R
Robert Olsson 已提交
644 645 646
	if (pkt_dev->flags & F_QUEUE_MAP_RND)
		seq_printf(seq,  "QUEUE_MAP_RND  ");

R
Robert Olsson 已提交
647 648 649
	if (pkt_dev->flags & F_QUEUE_MAP_CPU)
		seq_printf(seq,  "QUEUE_MAP_CPU  ");

650 651 652 653 654 655 656
	if (pkt_dev->cflows) {
		if (pkt_dev->flags & F_FLOW_SEQ)
			seq_printf(seq,  "FLOW_SEQ  "); /*in sequence flows*/
		else
			seq_printf(seq,  "FLOW_RND  ");
	}

J
Jamal Hadi Salim 已提交
657 658 659 660 661
#ifdef CONFIG_XFRM
	if (pkt_dev->flags & F_IPSEC_ON)
		seq_printf(seq,  "IPSEC  ");
#endif

L
Luiz Capitulino 已提交
662 663
	if (pkt_dev->flags & F_MACSRC_RND)
		seq_printf(seq, "MACSRC_RND  ");
L
Linus Torvalds 已提交
664

L
Luiz Capitulino 已提交
665 666
	if (pkt_dev->flags & F_MACDST_RND)
		seq_printf(seq, "MACDST_RND  ");
L
Linus Torvalds 已提交
667

F
Francesco Fondelli 已提交
668 669 670 671 672 673
	if (pkt_dev->flags & F_VID_RND)
		seq_printf(seq, "VID_RND  ");

	if (pkt_dev->flags & F_SVID_RND)
		seq_printf(seq, "SVID_RND  ");

R
Robert Olsson 已提交
674 675 676
	if (pkt_dev->flags & F_NODE)
		seq_printf(seq, "NODE_ALLOC  ");

L
Luiz Capitulino 已提交
677 678
	seq_puts(seq, "\n");

679 680 681 682
	/* not really stopped, more like last-running-at */
	stopped = pkt_dev->running ? ktime_now() : pkt_dev->stopped_at;
	idle = pkt_dev->idle_acc;
	do_div(idle, NSEC_PER_USEC);
L
Linus Torvalds 已提交
683

L
Luiz Capitulino 已提交
684
	seq_printf(seq,
685
		   "Current:\n     pkts-sofar: %llu  errors: %llu\n",
L
Luiz Capitulino 已提交
686
		   (unsigned long long)pkt_dev->sofar,
687 688 689 690 691 692 693
		   (unsigned long long)pkt_dev->errors);

	seq_printf(seq,
		   "     started: %lluus  stopped: %lluus idle: %lluus\n",
		   (unsigned long long) ktime_to_us(pkt_dev->started_at),
		   (unsigned long long) ktime_to_us(stopped),
		   (unsigned long long) idle);
L
Linus Torvalds 已提交
694

L
Luiz Capitulino 已提交
695 696
	seq_printf(seq,
		   "     seq_num: %d  cur_dst_mac_offset: %d  cur_src_mac_offset: %d\n",
697 698
		   pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset,
		   pkt_dev->cur_src_mac_offset);
L
Linus Torvalds 已提交
699

L
Luiz Capitulino 已提交
700
	if (pkt_dev->flags & F_IPV6) {
701 702 703
		seq_printf(seq, "     cur_saddr: %pI6c  cur_daddr: %pI6c\n",
				&pkt_dev->cur_in6_saddr,
				&pkt_dev->cur_in6_daddr);
L
Luiz Capitulino 已提交
704 705
	} else
		seq_printf(seq, "     cur_saddr: 0x%x  cur_daddr: 0x%x\n",
706
			   pkt_dev->cur_saddr, pkt_dev->cur_daddr);
L
Linus Torvalds 已提交
707

L
Luiz Capitulino 已提交
708
	seq_printf(seq, "     cur_udp_dst: %d  cur_udp_src: %d\n",
709
		   pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src);
L
Linus Torvalds 已提交
710

R
Robert Olsson 已提交
711 712
	seq_printf(seq, "     cur_queue_map: %u\n", pkt_dev->cur_queue_map);

L
Luiz Capitulino 已提交
713
	seq_printf(seq, "     flows: %u\n", pkt_dev->nflows);
L
Linus Torvalds 已提交
714 715

	if (pkt_dev->result[0])
L
Luiz Capitulino 已提交
716
		seq_printf(seq, "Result: %s\n", pkt_dev->result);
L
Linus Torvalds 已提交
717
	else
L
Luiz Capitulino 已提交
718
		seq_printf(seq, "Result: Idle\n");
L
Linus Torvalds 已提交
719

720
	return 0;
L
Linus Torvalds 已提交
721 722
}

723

724 725
static int hex32_arg(const char __user *user_buffer, unsigned long maxlen,
		     __u32 *num)
726 727 728 729
{
	int i = 0;
	*num = 0;

730
	for (; i < maxlen; i++) {
731
		int value;
732 733 734 735
		char c;
		*num <<= 4;
		if (get_user(c, &user_buffer[i]))
			return -EFAULT;
736 737 738
		value = hex_to_bin(c);
		if (value >= 0)
			*num |= value;
739 740 741 742 743 744
		else
			break;
	}
	return i;
}

L
Luiz Capitulino 已提交
745 746
static int count_trail_chars(const char __user * user_buffer,
			     unsigned int maxlen)
L
Linus Torvalds 已提交
747 748 749 750
{
	int i;

	for (i = 0; i < maxlen; i++) {
L
Luiz Capitulino 已提交
751 752 753 754
		char c;
		if (get_user(c, &user_buffer[i]))
			return -EFAULT;
		switch (c) {
L
Linus Torvalds 已提交
755 756 757 758 759 760 761 762 763
		case '\"':
		case '\n':
		case '\r':
		case '\t':
		case ' ':
		case '=':
			break;
		default:
			goto done;
764
		}
L
Linus Torvalds 已提交
765 766 767 768 769
	}
done:
	return i;
}

L
Luiz Capitulino 已提交
770 771
static unsigned long num_arg(const char __user * user_buffer,
			     unsigned long maxlen, unsigned long *num)
L
Linus Torvalds 已提交
772
{
773
	int i;
L
Linus Torvalds 已提交
774
	*num = 0;
L
Luiz Capitulino 已提交
775

776
	for (i = 0; i < maxlen; i++) {
L
Luiz Capitulino 已提交
777 778 779 780
		char c;
		if (get_user(c, &user_buffer[i]))
			return -EFAULT;
		if ((c >= '0') && (c <= '9')) {
L
Linus Torvalds 已提交
781
			*num *= 10;
L
Luiz Capitulino 已提交
782
			*num += c - '0';
L
Linus Torvalds 已提交
783 784 785 786 787 788
		} else
			break;
	}
	return i;
}

L
Luiz Capitulino 已提交
789
static int strn_len(const char __user * user_buffer, unsigned int maxlen)
L
Linus Torvalds 已提交
790
{
791
	int i;
L
Linus Torvalds 已提交
792

793
	for (i = 0; i < maxlen; i++) {
L
Luiz Capitulino 已提交
794 795 796 797
		char c;
		if (get_user(c, &user_buffer[i]))
			return -EFAULT;
		switch (c) {
L
Linus Torvalds 已提交
798 799 800 801 802 803 804 805 806
		case '\"':
		case '\n':
		case '\r':
		case '\t':
		case ' ':
			goto done_str;
			break;
		default:
			break;
807
		}
L
Linus Torvalds 已提交
808 809 810 811 812
	}
done_str:
	return i;
}

813 814 815 816 817 818 819 820 821 822
static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev)
{
	unsigned n = 0;
	char c;
	ssize_t i = 0;
	int len;

	pkt_dev->nr_labels = 0;
	do {
		__u32 tmp;
F
Francesco Fondelli 已提交
823
		len = hex32_arg(&buffer[i], 8, &tmp);
824 825 826 827 828 829 830 831 832 833 834 835
		if (len <= 0)
			return len;
		pkt_dev->labels[n] = htonl(tmp);
		if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM)
			pkt_dev->flags |= F_MPLS_RND;
		i += len;
		if (get_user(c, &buffer[i]))
			return -EFAULT;
		i++;
		n++;
		if (n >= MAX_MPLS_LABELS)
			return -E2BIG;
836
	} while (c == ',');
837 838 839 840 841

	pkt_dev->nr_labels = n;
	return i;
}

L
Luiz Capitulino 已提交
842 843 844
static ssize_t pktgen_if_write(struct file *file,
			       const char __user * user_buffer, size_t count,
			       loff_t * offset)
L
Linus Torvalds 已提交
845
{
846
	struct seq_file *seq = file->private_data;
L
Luiz Capitulino 已提交
847
	struct pktgen_dev *pkt_dev = seq->private;
848
	int i, max, len;
L
Linus Torvalds 已提交
849 850
	char name[16], valstr[32];
	unsigned long value = 0;
L
Luiz Capitulino 已提交
851 852
	char *pg_result = NULL;
	int tmp = 0;
L
Linus Torvalds 已提交
853
	char buf[128];
L
Luiz Capitulino 已提交
854 855 856

	pg_result = &(pkt_dev->result[0]);

L
Linus Torvalds 已提交
857
	if (count < 1) {
J
Joe Perches 已提交
858
		pr_warning("wrong command format\n");
L
Linus Torvalds 已提交
859 860
		return -EINVAL;
	}
L
Luiz Capitulino 已提交
861

862 863
	max = count;
	tmp = count_trail_chars(user_buffer, max);
L
Luiz Capitulino 已提交
864
	if (tmp < 0) {
J
Joe Perches 已提交
865
		pr_warning("illegal format\n");
L
Luiz Capitulino 已提交
866
		return tmp;
L
Linus Torvalds 已提交
867
	}
868
	i = tmp;
L
Luiz Capitulino 已提交
869

L
Linus Torvalds 已提交
870 871 872
	/* Read variable name */

	len = strn_len(&user_buffer[i], sizeof(name) - 1);
873
	if (len < 0)
L
Luiz Capitulino 已提交
874
		return len;
875

L
Linus Torvalds 已提交
876
	memset(name, 0, sizeof(name));
L
Luiz Capitulino 已提交
877
	if (copy_from_user(name, &user_buffer[i], len))
L
Linus Torvalds 已提交
878 879
		return -EFAULT;
	i += len;
L
Luiz Capitulino 已提交
880 881

	max = count - i;
L
Linus Torvalds 已提交
882
	len = count_trail_chars(&user_buffer[i], max);
L
Luiz Capitulino 已提交
883 884 885
	if (len < 0)
		return len;

L
Linus Torvalds 已提交
886 887 888
	i += len;

	if (debug) {
889
		size_t copy = min_t(size_t, count, 1023);
890 891
		char tb[copy + 1];
		if (copy_from_user(tb, user_buffer, copy))
L
Linus Torvalds 已提交
892
			return -EFAULT;
893
		tb[copy] = 0;
894
		printk(KERN_DEBUG "pktgen: %s,%lu  buffer -:%s:-\n", name,
L
Luiz Capitulino 已提交
895 896
		       (unsigned long)count, tb);
	}
L
Linus Torvalds 已提交
897 898 899

	if (!strcmp(name, "min_pkt_size")) {
		len = num_arg(&user_buffer[i], 10, &value);
900
		if (len < 0)
L
Luiz Capitulino 已提交
901
			return len;
902

L
Linus Torvalds 已提交
903
		i += len;
L
Luiz Capitulino 已提交
904 905 906 907 908 909 910 911
		if (value < 14 + 20 + 8)
			value = 14 + 20 + 8;
		if (value != pkt_dev->min_pkt_size) {
			pkt_dev->min_pkt_size = value;
			pkt_dev->cur_pkt_size = value;
		}
		sprintf(pg_result, "OK: min_pkt_size=%u",
			pkt_dev->min_pkt_size);
L
Linus Torvalds 已提交
912 913 914
		return count;
	}

L
Luiz Capitulino 已提交
915
	if (!strcmp(name, "max_pkt_size")) {
L
Linus Torvalds 已提交
916
		len = num_arg(&user_buffer[i], 10, &value);
917
		if (len < 0)
L
Luiz Capitulino 已提交
918
			return len;
919

L
Linus Torvalds 已提交
920
		i += len;
L
Luiz Capitulino 已提交
921 922 923 924 925 926 927 928
		if (value < 14 + 20 + 8)
			value = 14 + 20 + 8;
		if (value != pkt_dev->max_pkt_size) {
			pkt_dev->max_pkt_size = value;
			pkt_dev->cur_pkt_size = value;
		}
		sprintf(pg_result, "OK: max_pkt_size=%u",
			pkt_dev->max_pkt_size);
L
Linus Torvalds 已提交
929 930 931
		return count;
	}

L
Luiz Capitulino 已提交
932
	/* Shortcut for min = max */
L
Linus Torvalds 已提交
933 934 935

	if (!strcmp(name, "pkt_size")) {
		len = num_arg(&user_buffer[i], 10, &value);
936
		if (len < 0)
L
Luiz Capitulino 已提交
937
			return len;
938

L
Linus Torvalds 已提交
939
		i += len;
L
Luiz Capitulino 已提交
940 941 942 943 944 945 946
		if (value < 14 + 20 + 8)
			value = 14 + 20 + 8;
		if (value != pkt_dev->min_pkt_size) {
			pkt_dev->min_pkt_size = value;
			pkt_dev->max_pkt_size = value;
			pkt_dev->cur_pkt_size = value;
		}
L
Linus Torvalds 已提交
947 948 949 950
		sprintf(pg_result, "OK: pkt_size=%u", pkt_dev->min_pkt_size);
		return count;
	}

L
Luiz Capitulino 已提交
951
	if (!strcmp(name, "debug")) {
L
Linus Torvalds 已提交
952
		len = num_arg(&user_buffer[i], 10, &value);
953
		if (len < 0)
L
Luiz Capitulino 已提交
954
			return len;
955

L
Linus Torvalds 已提交
956
		i += len;
L
Luiz Capitulino 已提交
957
		debug = value;
L
Linus Torvalds 已提交
958 959 960 961
		sprintf(pg_result, "OK: debug=%u", debug);
		return count;
	}

L
Luiz Capitulino 已提交
962
	if (!strcmp(name, "frags")) {
L
Linus Torvalds 已提交
963
		len = num_arg(&user_buffer[i], 10, &value);
964
		if (len < 0)
L
Luiz Capitulino 已提交
965
			return len;
966

L
Linus Torvalds 已提交
967 968 969 970 971 972 973
		i += len;
		pkt_dev->nfrags = value;
		sprintf(pg_result, "OK: frags=%u", pkt_dev->nfrags);
		return count;
	}
	if (!strcmp(name, "delay")) {
		len = num_arg(&user_buffer[i], 10, &value);
974
		if (len < 0)
L
Luiz Capitulino 已提交
975
			return len;
976

L
Linus Torvalds 已提交
977
		i += len;
978 979 980
		if (value == 0x7FFFFFFF)
			pkt_dev->delay = ULLONG_MAX;
		else
E
Eric Dumazet 已提交
981
			pkt_dev->delay = (u64)value;
982 983 984

		sprintf(pg_result, "OK: delay=%llu",
			(unsigned long long) pkt_dev->delay);
L
Linus Torvalds 已提交
985 986
		return count;
	}
987 988 989 990 991 992 993 994 995 996
	if (!strcmp(name, "rate")) {
		len = num_arg(&user_buffer[i], 10, &value);
		if (len < 0)
			return len;

		i += len;
		if (!value)
			return len;
		pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value;
		if (debug)
J
Joe Perches 已提交
997
			pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011

		sprintf(pg_result, "OK: rate=%lu", value);
		return count;
	}
	if (!strcmp(name, "ratep")) {
		len = num_arg(&user_buffer[i], 10, &value);
		if (len < 0)
			return len;

		i += len;
		if (!value)
			return len;
		pkt_dev->delay = NSEC_PER_SEC/value;
		if (debug)
J
Joe Perches 已提交
1012
			pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1013 1014 1015 1016

		sprintf(pg_result, "OK: rate=%lu", value);
		return count;
	}
L
Luiz Capitulino 已提交
1017
	if (!strcmp(name, "udp_src_min")) {
L
Linus Torvalds 已提交
1018
		len = num_arg(&user_buffer[i], 10, &value);
1019
		if (len < 0)
L
Luiz Capitulino 已提交
1020
			return len;
1021

L
Linus Torvalds 已提交
1022
		i += len;
L
Luiz Capitulino 已提交
1023 1024 1025 1026
		if (value != pkt_dev->udp_src_min) {
			pkt_dev->udp_src_min = value;
			pkt_dev->cur_udp_src = value;
		}
L
Linus Torvalds 已提交
1027 1028 1029
		sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min);
		return count;
	}
L
Luiz Capitulino 已提交
1030
	if (!strcmp(name, "udp_dst_min")) {
L
Linus Torvalds 已提交
1031
		len = num_arg(&user_buffer[i], 10, &value);
1032
		if (len < 0)
L
Luiz Capitulino 已提交
1033
			return len;
1034

L
Linus Torvalds 已提交
1035
		i += len;
L
Luiz Capitulino 已提交
1036 1037 1038 1039
		if (value != pkt_dev->udp_dst_min) {
			pkt_dev->udp_dst_min = value;
			pkt_dev->cur_udp_dst = value;
		}
L
Linus Torvalds 已提交
1040 1041 1042
		sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min);
		return count;
	}
L
Luiz Capitulino 已提交
1043
	if (!strcmp(name, "udp_src_max")) {
L
Linus Torvalds 已提交
1044
		len = num_arg(&user_buffer[i], 10, &value);
1045
		if (len < 0)
L
Luiz Capitulino 已提交
1046
			return len;
1047

L
Linus Torvalds 已提交
1048
		i += len;
L
Luiz Capitulino 已提交
1049 1050 1051 1052
		if (value != pkt_dev->udp_src_max) {
			pkt_dev->udp_src_max = value;
			pkt_dev->cur_udp_src = value;
		}
L
Linus Torvalds 已提交
1053 1054 1055
		sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max);
		return count;
	}
L
Luiz Capitulino 已提交
1056
	if (!strcmp(name, "udp_dst_max")) {
L
Linus Torvalds 已提交
1057
		len = num_arg(&user_buffer[i], 10, &value);
1058
		if (len < 0)
L
Luiz Capitulino 已提交
1059
			return len;
1060

L
Linus Torvalds 已提交
1061
		i += len;
L
Luiz Capitulino 已提交
1062 1063 1064 1065
		if (value != pkt_dev->udp_dst_max) {
			pkt_dev->udp_dst_max = value;
			pkt_dev->cur_udp_dst = value;
		}
L
Linus Torvalds 已提交
1066 1067 1068 1069 1070
		sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max);
		return count;
	}
	if (!strcmp(name, "clone_skb")) {
		len = num_arg(&user_buffer[i], 10, &value);
1071
		if (len < 0)
L
Luiz Capitulino 已提交
1072
			return len;
1073 1074 1075
		if ((value > 0) &&
		    (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))
			return -ENOTSUPP;
L
Linus Torvalds 已提交
1076
		i += len;
L
Luiz Capitulino 已提交
1077 1078
		pkt_dev->clone_skb = value;

L
Linus Torvalds 已提交
1079 1080 1081 1082 1083
		sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb);
		return count;
	}
	if (!strcmp(name, "count")) {
		len = num_arg(&user_buffer[i], 10, &value);
1084
		if (len < 0)
L
Luiz Capitulino 已提交
1085
			return len;
1086

L
Linus Torvalds 已提交
1087 1088 1089
		i += len;
		pkt_dev->count = value;
		sprintf(pg_result, "OK: count=%llu",
L
Luiz Capitulino 已提交
1090
			(unsigned long long)pkt_dev->count);
L
Linus Torvalds 已提交
1091 1092 1093 1094
		return count;
	}
	if (!strcmp(name, "src_mac_count")) {
		len = num_arg(&user_buffer[i], 10, &value);
1095
		if (len < 0)
L
Luiz Capitulino 已提交
1096
			return len;
1097

L
Linus Torvalds 已提交
1098 1099
		i += len;
		if (pkt_dev->src_mac_count != value) {
L
Luiz Capitulino 已提交
1100 1101 1102 1103 1104
			pkt_dev->src_mac_count = value;
			pkt_dev->cur_src_mac_offset = 0;
		}
		sprintf(pg_result, "OK: src_mac_count=%d",
			pkt_dev->src_mac_count);
L
Linus Torvalds 已提交
1105 1106 1107 1108
		return count;
	}
	if (!strcmp(name, "dst_mac_count")) {
		len = num_arg(&user_buffer[i], 10, &value);
1109
		if (len < 0)
L
Luiz Capitulino 已提交
1110
			return len;
1111

L
Linus Torvalds 已提交
1112 1113
		i += len;
		if (pkt_dev->dst_mac_count != value) {
L
Luiz Capitulino 已提交
1114 1115 1116 1117 1118
			pkt_dev->dst_mac_count = value;
			pkt_dev->cur_dst_mac_offset = 0;
		}
		sprintf(pg_result, "OK: dst_mac_count=%d",
			pkt_dev->dst_mac_count);
L
Linus Torvalds 已提交
1119 1120
		return count;
	}
R
Robert Olsson 已提交
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
	if (!strcmp(name, "node")) {
		len = num_arg(&user_buffer[i], 10, &value);
		if (len < 0)
			return len;

		i += len;

		if (node_possible(value)) {
			pkt_dev->node = value;
			sprintf(pg_result, "OK: node=%d", pkt_dev->node);
E
Eric Dumazet 已提交
1131 1132 1133 1134
			if (pkt_dev->page) {
				put_page(pkt_dev->page);
				pkt_dev->page = NULL;
			}
R
Robert Olsson 已提交
1135 1136 1137 1138 1139
		}
		else
			sprintf(pg_result, "ERROR: node not possible");
		return count;
	}
L
Linus Torvalds 已提交
1140
	if (!strcmp(name, "flag")) {
L
Luiz Capitulino 已提交
1141 1142
		char f[32];
		memset(f, 0, 32);
L
Linus Torvalds 已提交
1143
		len = strn_len(&user_buffer[i], sizeof(f) - 1);
1144
		if (len < 0)
L
Luiz Capitulino 已提交
1145
			return len;
1146

L
Linus Torvalds 已提交
1147 1148 1149
		if (copy_from_user(f, &user_buffer[i], len))
			return -EFAULT;
		i += len;
L
Luiz Capitulino 已提交
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
		if (strcmp(f, "IPSRC_RND") == 0)
			pkt_dev->flags |= F_IPSRC_RND;

		else if (strcmp(f, "!IPSRC_RND") == 0)
			pkt_dev->flags &= ~F_IPSRC_RND;

		else if (strcmp(f, "TXSIZE_RND") == 0)
			pkt_dev->flags |= F_TXSIZE_RND;

		else if (strcmp(f, "!TXSIZE_RND") == 0)
			pkt_dev->flags &= ~F_TXSIZE_RND;

		else if (strcmp(f, "IPDST_RND") == 0)
			pkt_dev->flags |= F_IPDST_RND;

		else if (strcmp(f, "!IPDST_RND") == 0)
			pkt_dev->flags &= ~F_IPDST_RND;

		else if (strcmp(f, "UDPSRC_RND") == 0)
			pkt_dev->flags |= F_UDPSRC_RND;

		else if (strcmp(f, "!UDPSRC_RND") == 0)
			pkt_dev->flags &= ~F_UDPSRC_RND;

		else if (strcmp(f, "UDPDST_RND") == 0)
			pkt_dev->flags |= F_UDPDST_RND;

		else if (strcmp(f, "!UDPDST_RND") == 0)
			pkt_dev->flags &= ~F_UDPDST_RND;

		else if (strcmp(f, "MACSRC_RND") == 0)
			pkt_dev->flags |= F_MACSRC_RND;

		else if (strcmp(f, "!MACSRC_RND") == 0)
			pkt_dev->flags &= ~F_MACSRC_RND;

		else if (strcmp(f, "MACDST_RND") == 0)
			pkt_dev->flags |= F_MACDST_RND;

		else if (strcmp(f, "!MACDST_RND") == 0)
			pkt_dev->flags &= ~F_MACDST_RND;

1192 1193 1194 1195 1196 1197
		else if (strcmp(f, "MPLS_RND") == 0)
			pkt_dev->flags |= F_MPLS_RND;

		else if (strcmp(f, "!MPLS_RND") == 0)
			pkt_dev->flags &= ~F_MPLS_RND;

F
Francesco Fondelli 已提交
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
		else if (strcmp(f, "VID_RND") == 0)
			pkt_dev->flags |= F_VID_RND;

		else if (strcmp(f, "!VID_RND") == 0)
			pkt_dev->flags &= ~F_VID_RND;

		else if (strcmp(f, "SVID_RND") == 0)
			pkt_dev->flags |= F_SVID_RND;

		else if (strcmp(f, "!SVID_RND") == 0)
			pkt_dev->flags &= ~F_SVID_RND;

1210 1211 1212
		else if (strcmp(f, "FLOW_SEQ") == 0)
			pkt_dev->flags |= F_FLOW_SEQ;

R
Robert Olsson 已提交
1213 1214 1215 1216 1217
		else if (strcmp(f, "QUEUE_MAP_RND") == 0)
			pkt_dev->flags |= F_QUEUE_MAP_RND;

		else if (strcmp(f, "!QUEUE_MAP_RND") == 0)
			pkt_dev->flags &= ~F_QUEUE_MAP_RND;
R
Robert Olsson 已提交
1218 1219 1220 1221 1222 1223

		else if (strcmp(f, "QUEUE_MAP_CPU") == 0)
			pkt_dev->flags |= F_QUEUE_MAP_CPU;

		else if (strcmp(f, "!QUEUE_MAP_CPU") == 0)
			pkt_dev->flags &= ~F_QUEUE_MAP_CPU;
J
Jamal Hadi Salim 已提交
1224 1225 1226 1227 1228
#ifdef CONFIG_XFRM
		else if (strcmp(f, "IPSEC") == 0)
			pkt_dev->flags |= F_IPSEC_ON;
#endif

F
Francesco Fondelli 已提交
1229 1230 1231
		else if (strcmp(f, "!IPV6") == 0)
			pkt_dev->flags &= ~F_IPV6;

R
Robert Olsson 已提交
1232 1233 1234 1235 1236 1237
		else if (strcmp(f, "NODE_ALLOC") == 0)
			pkt_dev->flags |= F_NODE;

		else if (strcmp(f, "!NODE_ALLOC") == 0)
			pkt_dev->flags &= ~F_NODE;

L
Luiz Capitulino 已提交
1238 1239 1240 1241
		else {
			sprintf(pg_result,
				"Flag -:%s:- unknown\nAvailable flags, (prepend ! to un-set flag):\n%s",
				f,
F
Francesco Fondelli 已提交
1242
				"IPSRC_RND, IPDST_RND, UDPSRC_RND, UDPDST_RND, "
R
Robert Olsson 已提交
1243
				"MACSRC_RND, MACDST_RND, TXSIZE_RND, IPV6, MPLS_RND, VID_RND, SVID_RND, FLOW_SEQ, IPSEC, NODE_ALLOC\n");
L
Luiz Capitulino 已提交
1244 1245
			return count;
		}
L
Linus Torvalds 已提交
1246 1247 1248 1249 1250
		sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags);
		return count;
	}
	if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) {
		len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_min) - 1);
1251
		if (len < 0)
L
Luiz Capitulino 已提交
1252
			return len;
L
Linus Torvalds 已提交
1253

L
Luiz Capitulino 已提交
1254
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1255
			return -EFAULT;
L
Luiz Capitulino 已提交
1256 1257 1258 1259 1260 1261 1262 1263
		buf[len] = 0;
		if (strcmp(buf, pkt_dev->dst_min) != 0) {
			memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min));
			strncpy(pkt_dev->dst_min, buf, len);
			pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
			pkt_dev->cur_daddr = pkt_dev->daddr_min;
		}
		if (debug)
1264
			printk(KERN_DEBUG "pktgen: dst_min set to: %s\n",
L
Luiz Capitulino 已提交
1265 1266
			       pkt_dev->dst_min);
		i += len;
L
Linus Torvalds 已提交
1267 1268 1269 1270 1271
		sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min);
		return count;
	}
	if (!strcmp(name, "dst_max")) {
		len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_max) - 1);
1272
		if (len < 0)
L
Luiz Capitulino 已提交
1273
			return len;
1274

L
Linus Torvalds 已提交
1275

L
Luiz Capitulino 已提交
1276
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1277 1278
			return -EFAULT;

L
Luiz Capitulino 已提交
1279 1280 1281 1282 1283 1284 1285 1286
		buf[len] = 0;
		if (strcmp(buf, pkt_dev->dst_max) != 0) {
			memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max));
			strncpy(pkt_dev->dst_max, buf, len);
			pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
			pkt_dev->cur_daddr = pkt_dev->daddr_max;
		}
		if (debug)
1287
			printk(KERN_DEBUG "pktgen: dst_max set to: %s\n",
L
Luiz Capitulino 已提交
1288
			       pkt_dev->dst_max);
L
Linus Torvalds 已提交
1289 1290 1291 1292 1293 1294
		i += len;
		sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max);
		return count;
	}
	if (!strcmp(name, "dst6")) {
		len = strn_len(&user_buffer[i], sizeof(buf) - 1);
L
Luiz Capitulino 已提交
1295 1296
		if (len < 0)
			return len;
L
Linus Torvalds 已提交
1297 1298 1299

		pkt_dev->flags |= F_IPV6;

L
Luiz Capitulino 已提交
1300
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1301
			return -EFAULT;
L
Luiz Capitulino 已提交
1302
		buf[len] = 0;
L
Linus Torvalds 已提交
1303 1304

		scan_ip6(buf, pkt_dev->in6_daddr.s6_addr);
1305
		snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr);
L
Linus Torvalds 已提交
1306 1307 1308

		ipv6_addr_copy(&pkt_dev->cur_in6_daddr, &pkt_dev->in6_daddr);

L
Luiz Capitulino 已提交
1309
		if (debug)
1310
			printk(KERN_DEBUG "pktgen: dst6 set to: %s\n", buf);
L
Linus Torvalds 已提交
1311

L
Luiz Capitulino 已提交
1312
		i += len;
L
Linus Torvalds 已提交
1313 1314 1315 1316 1317
		sprintf(pg_result, "OK: dst6=%s", buf);
		return count;
	}
	if (!strcmp(name, "dst6_min")) {
		len = strn_len(&user_buffer[i], sizeof(buf) - 1);
L
Luiz Capitulino 已提交
1318 1319
		if (len < 0)
			return len;
L
Linus Torvalds 已提交
1320 1321 1322

		pkt_dev->flags |= F_IPV6;

L
Luiz Capitulino 已提交
1323
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1324
			return -EFAULT;
L
Luiz Capitulino 已提交
1325
		buf[len] = 0;
L
Linus Torvalds 已提交
1326 1327

		scan_ip6(buf, pkt_dev->min_in6_daddr.s6_addr);
1328
		snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr);
L
Linus Torvalds 已提交
1329

L
Luiz Capitulino 已提交
1330 1331 1332
		ipv6_addr_copy(&pkt_dev->cur_in6_daddr,
			       &pkt_dev->min_in6_daddr);
		if (debug)
1333
			printk(KERN_DEBUG "pktgen: dst6_min set to: %s\n", buf);
L
Linus Torvalds 已提交
1334

L
Luiz Capitulino 已提交
1335
		i += len;
L
Linus Torvalds 已提交
1336 1337 1338 1339 1340
		sprintf(pg_result, "OK: dst6_min=%s", buf);
		return count;
	}
	if (!strcmp(name, "dst6_max")) {
		len = strn_len(&user_buffer[i], sizeof(buf) - 1);
L
Luiz Capitulino 已提交
1341 1342
		if (len < 0)
			return len;
L
Linus Torvalds 已提交
1343 1344 1345

		pkt_dev->flags |= F_IPV6;

L
Luiz Capitulino 已提交
1346
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1347
			return -EFAULT;
L
Luiz Capitulino 已提交
1348
		buf[len] = 0;
L
Linus Torvalds 已提交
1349 1350

		scan_ip6(buf, pkt_dev->max_in6_daddr.s6_addr);
1351
		snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr);
L
Linus Torvalds 已提交
1352

L
Luiz Capitulino 已提交
1353
		if (debug)
1354
			printk(KERN_DEBUG "pktgen: dst6_max set to: %s\n", buf);
L
Linus Torvalds 已提交
1355

L
Luiz Capitulino 已提交
1356
		i += len;
L
Linus Torvalds 已提交
1357 1358 1359 1360 1361
		sprintf(pg_result, "OK: dst6_max=%s", buf);
		return count;
	}
	if (!strcmp(name, "src6")) {
		len = strn_len(&user_buffer[i], sizeof(buf) - 1);
L
Luiz Capitulino 已提交
1362 1363
		if (len < 0)
			return len;
L
Linus Torvalds 已提交
1364 1365 1366

		pkt_dev->flags |= F_IPV6;

L
Luiz Capitulino 已提交
1367
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1368
			return -EFAULT;
L
Luiz Capitulino 已提交
1369
		buf[len] = 0;
L
Linus Torvalds 已提交
1370 1371

		scan_ip6(buf, pkt_dev->in6_saddr.s6_addr);
1372
		snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr);
L
Linus Torvalds 已提交
1373 1374 1375

		ipv6_addr_copy(&pkt_dev->cur_in6_saddr, &pkt_dev->in6_saddr);

L
Luiz Capitulino 已提交
1376
		if (debug)
1377
			printk(KERN_DEBUG "pktgen: src6 set to: %s\n", buf);
L
Luiz Capitulino 已提交
1378 1379

		i += len;
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384
		sprintf(pg_result, "OK: src6=%s", buf);
		return count;
	}
	if (!strcmp(name, "src_min")) {
		len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_min) - 1);
1385
		if (len < 0)
L
Luiz Capitulino 已提交
1386
			return len;
1387

L
Luiz Capitulino 已提交
1388
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1389
			return -EFAULT;
L
Luiz Capitulino 已提交
1390 1391 1392 1393 1394 1395 1396 1397
		buf[len] = 0;
		if (strcmp(buf, pkt_dev->src_min) != 0) {
			memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min));
			strncpy(pkt_dev->src_min, buf, len);
			pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
			pkt_dev->cur_saddr = pkt_dev->saddr_min;
		}
		if (debug)
1398
			printk(KERN_DEBUG "pktgen: src_min set to: %s\n",
L
Luiz Capitulino 已提交
1399
			       pkt_dev->src_min);
L
Linus Torvalds 已提交
1400 1401 1402 1403 1404 1405
		i += len;
		sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min);
		return count;
	}
	if (!strcmp(name, "src_max")) {
		len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_max) - 1);
1406
		if (len < 0)
L
Luiz Capitulino 已提交
1407
			return len;
1408

L
Luiz Capitulino 已提交
1409
		if (copy_from_user(buf, &user_buffer[i], len))
L
Linus Torvalds 已提交
1410
			return -EFAULT;
L
Luiz Capitulino 已提交
1411 1412 1413 1414 1415 1416 1417 1418
		buf[len] = 0;
		if (strcmp(buf, pkt_dev->src_max) != 0) {
			memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max));
			strncpy(pkt_dev->src_max, buf, len);
			pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
			pkt_dev->cur_saddr = pkt_dev->saddr_max;
		}
		if (debug)
1419
			printk(KERN_DEBUG "pktgen: src_max set to: %s\n",
L
Luiz Capitulino 已提交
1420
			       pkt_dev->src_max);
L
Linus Torvalds 已提交
1421 1422 1423 1424 1425 1426
		i += len;
		sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max);
		return count;
	}
	if (!strcmp(name, "dst_mac")) {
		len = strn_len(&user_buffer[i], sizeof(valstr) - 1);
1427
		if (len < 0)
L
Luiz Capitulino 已提交
1428
			return len;
1429

L
Linus Torvalds 已提交
1430
		memset(valstr, 0, sizeof(valstr));
L
Luiz Capitulino 已提交
1431
		if (copy_from_user(valstr, &user_buffer[i], len))
L
Linus Torvalds 已提交
1432 1433
			return -EFAULT;

1434 1435
		if (!mac_pton(valstr, pkt_dev->dst_mac))
			return -EINVAL;
L
Linus Torvalds 已提交
1436
		/* Set up Dest MAC */
1437
		memcpy(&pkt_dev->hh[0], pkt_dev->dst_mac, ETH_ALEN);
L
Luiz Capitulino 已提交
1438

1439
		sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac);
L
Linus Torvalds 已提交
1440 1441 1442 1443
		return count;
	}
	if (!strcmp(name, "src_mac")) {
		len = strn_len(&user_buffer[i], sizeof(valstr) - 1);
1444
		if (len < 0)
L
Luiz Capitulino 已提交
1445
			return len;
1446

L
Linus Torvalds 已提交
1447
		memset(valstr, 0, sizeof(valstr));
L
Luiz Capitulino 已提交
1448
		if (copy_from_user(valstr, &user_buffer[i], len))
L
Linus Torvalds 已提交
1449 1450
			return -EFAULT;

1451 1452
		if (!mac_pton(valstr, pkt_dev->src_mac))
			return -EINVAL;
A
Adit Ranadive 已提交
1453
		/* Set up Src MAC */
1454
		memcpy(&pkt_dev->hh[6], pkt_dev->src_mac, ETH_ALEN);
A
Adit Ranadive 已提交
1455

1456
		sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac);
L
Linus Torvalds 已提交
1457 1458 1459
		return count;
	}

L
Luiz Capitulino 已提交
1460 1461 1462 1463 1464
	if (!strcmp(name, "clear_counters")) {
		pktgen_clear_counters(pkt_dev);
		sprintf(pg_result, "OK: Clearing counters.\n");
		return count;
	}
L
Linus Torvalds 已提交
1465 1466 1467

	if (!strcmp(name, "flows")) {
		len = num_arg(&user_buffer[i], 10, &value);
1468
		if (len < 0)
L
Luiz Capitulino 已提交
1469
			return len;
1470

L
Linus Torvalds 已提交
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
		i += len;
		if (value > MAX_CFLOWS)
			value = MAX_CFLOWS;

		pkt_dev->cflows = value;
		sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows);
		return count;
	}

	if (!strcmp(name, "flowlen")) {
		len = num_arg(&user_buffer[i], 10, &value);
1482
		if (len < 0)
L
Luiz Capitulino 已提交
1483
			return len;
1484

L
Linus Torvalds 已提交
1485 1486 1487 1488 1489
		i += len;
		pkt_dev->lflow = value;
		sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow);
		return count;
	}
L
Luiz Capitulino 已提交
1490

R
Robert Olsson 已提交
1491 1492
	if (!strcmp(name, "queue_map_min")) {
		len = num_arg(&user_buffer[i], 5, &value);
1493
		if (len < 0)
R
Robert Olsson 已提交
1494
			return len;
1495

R
Robert Olsson 已提交
1496 1497 1498 1499 1500 1501 1502 1503
		i += len;
		pkt_dev->queue_map_min = value;
		sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min);
		return count;
	}

	if (!strcmp(name, "queue_map_max")) {
		len = num_arg(&user_buffer[i], 5, &value);
1504
		if (len < 0)
R
Robert Olsson 已提交
1505
			return len;
1506

R
Robert Olsson 已提交
1507 1508 1509 1510 1511 1512
		i += len;
		pkt_dev->queue_map_max = value;
		sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max);
		return count;
	}

1513
	if (!strcmp(name, "mpls")) {
S
Stephen Hemminger 已提交
1514 1515
		unsigned n, cnt;

1516
		len = get_labels(&user_buffer[i], pkt_dev);
S
Stephen Hemminger 已提交
1517 1518
		if (len < 0)
			return len;
1519
		i += len;
S
Stephen Hemminger 已提交
1520
		cnt = sprintf(pg_result, "OK: mpls=");
1521
		for (n = 0; n < pkt_dev->nr_labels; n++)
S
Stephen Hemminger 已提交
1522 1523 1524
			cnt += sprintf(pg_result + cnt,
				       "%08x%s", ntohl(pkt_dev->labels[n]),
				       n == pkt_dev->nr_labels-1 ? "" : ",");
F
Francesco Fondelli 已提交
1525 1526 1527 1528 1529 1530

		if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) {
			pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
			pkt_dev->svlan_id = 0xffff;

			if (debug)
1531
				printk(KERN_DEBUG "pktgen: VLAN/SVLAN auto turned off\n");
F
Francesco Fondelli 已提交
1532 1533 1534 1535 1536 1537
		}
		return count;
	}

	if (!strcmp(name, "vlan_id")) {
		len = num_arg(&user_buffer[i], 4, &value);
1538
		if (len < 0)
F
Francesco Fondelli 已提交
1539
			return len;
1540

F
Francesco Fondelli 已提交
1541 1542 1543 1544 1545
		i += len;
		if (value <= 4095) {
			pkt_dev->vlan_id = value;  /* turn on VLAN */

			if (debug)
1546
				printk(KERN_DEBUG "pktgen: VLAN turned on\n");
F
Francesco Fondelli 已提交
1547 1548

			if (debug && pkt_dev->nr_labels)
1549
				printk(KERN_DEBUG "pktgen: MPLS auto turned off\n");
F
Francesco Fondelli 已提交
1550 1551 1552 1553 1554 1555 1556 1557

			pkt_dev->nr_labels = 0;    /* turn off MPLS */
			sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id);
		} else {
			pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
			pkt_dev->svlan_id = 0xffff;

			if (debug)
1558
				printk(KERN_DEBUG "pktgen: VLAN/SVLAN turned off\n");
F
Francesco Fondelli 已提交
1559 1560 1561 1562 1563 1564
		}
		return count;
	}

	if (!strcmp(name, "vlan_p")) {
		len = num_arg(&user_buffer[i], 1, &value);
1565
		if (len < 0)
F
Francesco Fondelli 已提交
1566
			return len;
1567

F
Francesco Fondelli 已提交
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
		i += len;
		if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) {
			pkt_dev->vlan_p = value;
			sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p);
		} else {
			sprintf(pg_result, "ERROR: vlan_p must be 0-7");
		}
		return count;
	}

	if (!strcmp(name, "vlan_cfi")) {
		len = num_arg(&user_buffer[i], 1, &value);
1580
		if (len < 0)
F
Francesco Fondelli 已提交
1581
			return len;
1582

F
Francesco Fondelli 已提交
1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
		i += len;
		if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) {
			pkt_dev->vlan_cfi = value;
			sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi);
		} else {
			sprintf(pg_result, "ERROR: vlan_cfi must be 0-1");
		}
		return count;
	}

	if (!strcmp(name, "svlan_id")) {
		len = num_arg(&user_buffer[i], 4, &value);
1595
		if (len < 0)
F
Francesco Fondelli 已提交
1596
			return len;
1597

F
Francesco Fondelli 已提交
1598 1599 1600 1601 1602
		i += len;
		if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) {
			pkt_dev->svlan_id = value;  /* turn on SVLAN */

			if (debug)
1603
				printk(KERN_DEBUG "pktgen: SVLAN turned on\n");
F
Francesco Fondelli 已提交
1604 1605

			if (debug && pkt_dev->nr_labels)
1606
				printk(KERN_DEBUG "pktgen: MPLS auto turned off\n");
F
Francesco Fondelli 已提交
1607 1608 1609 1610 1611 1612 1613 1614

			pkt_dev->nr_labels = 0;    /* turn off MPLS */
			sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id);
		} else {
			pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
			pkt_dev->svlan_id = 0xffff;

			if (debug)
1615
				printk(KERN_DEBUG "pktgen: VLAN/SVLAN turned off\n");
F
Francesco Fondelli 已提交
1616 1617 1618 1619 1620 1621
		}
		return count;
	}

	if (!strcmp(name, "svlan_p")) {
		len = num_arg(&user_buffer[i], 1, &value);
1622
		if (len < 0)
F
Francesco Fondelli 已提交
1623
			return len;
1624

F
Francesco Fondelli 已提交
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
		i += len;
		if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) {
			pkt_dev->svlan_p = value;
			sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p);
		} else {
			sprintf(pg_result, "ERROR: svlan_p must be 0-7");
		}
		return count;
	}

	if (!strcmp(name, "svlan_cfi")) {
		len = num_arg(&user_buffer[i], 1, &value);
1637
		if (len < 0)
F
Francesco Fondelli 已提交
1638
			return len;
1639

F
Francesco Fondelli 已提交
1640 1641 1642 1643 1644 1645 1646
		i += len;
		if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) {
			pkt_dev->svlan_cfi = value;
			sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi);
		} else {
			sprintf(pg_result, "ERROR: svlan_cfi must be 0-1");
		}
1647 1648 1649
		return count;
	}

F
Francesco Fondelli 已提交
1650 1651 1652
	if (!strcmp(name, "tos")) {
		__u32 tmp_value = 0;
		len = hex32_arg(&user_buffer[i], 2, &tmp_value);
1653
		if (len < 0)
F
Francesco Fondelli 已提交
1654
			return len;
1655

F
Francesco Fondelli 已提交
1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
		i += len;
		if (len == 2) {
			pkt_dev->tos = tmp_value;
			sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos);
		} else {
			sprintf(pg_result, "ERROR: tos must be 00-ff");
		}
		return count;
	}

	if (!strcmp(name, "traffic_class")) {
		__u32 tmp_value = 0;
		len = hex32_arg(&user_buffer[i], 2, &tmp_value);
1669
		if (len < 0)
F
Francesco Fondelli 已提交
1670
			return len;
1671

F
Francesco Fondelli 已提交
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681
		i += len;
		if (len == 2) {
			pkt_dev->traffic_class = tmp_value;
			sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class);
		} else {
			sprintf(pg_result, "ERROR: traffic_class must be 00-ff");
		}
		return count;
	}

J
John Fastabend 已提交
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
	if (!strcmp(name, "skb_priority")) {
		len = num_arg(&user_buffer[i], 9, &value);
		if (len < 0)
			return len;

		i += len;
		pkt_dev->skb_priority = value;
		sprintf(pg_result, "OK: skb_priority=%i",
			pkt_dev->skb_priority);
		return count;
	}

L
Linus Torvalds 已提交
1694 1695 1696 1697
	sprintf(pkt_dev->result, "No such parameter \"%s\"", name);
	return -EINVAL;
}

1698
static int pktgen_if_open(struct inode *inode, struct file *file)
L
Linus Torvalds 已提交
1699
{
1700 1701
	return single_open(file, pktgen_if_show, PDE(inode)->data);
}
L
Linus Torvalds 已提交
1702

1703
static const struct file_operations pktgen_if_fops = {
L
Luiz Capitulino 已提交
1704 1705 1706 1707 1708 1709
	.owner   = THIS_MODULE,
	.open    = pktgen_if_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.write   = pktgen_if_write,
	.release = single_release,
1710
};
L
Linus Torvalds 已提交
1711

1712 1713
static int pktgen_thread_show(struct seq_file *seq, void *v)
{
L
Luiz Capitulino 已提交
1714
	struct pktgen_thread *t = seq->private;
S
Stephen Hemminger 已提交
1715
	const struct pktgen_dev *pkt_dev;
1716 1717

	BUG_ON(!t);
L
Linus Torvalds 已提交
1718

L
Luiz Capitulino 已提交
1719
	seq_printf(seq, "Running: ");
L
Linus Torvalds 已提交
1720

L
Luiz Capitulino 已提交
1721
	if_lock(t);
1722
	list_for_each_entry(pkt_dev, &t->if_list, list)
L
Luiz Capitulino 已提交
1723
		if (pkt_dev->running)
E
Eric Dumazet 已提交
1724
			seq_printf(seq, "%s ", pkt_dev->odevname);
L
Linus Torvalds 已提交
1725

L
Luiz Capitulino 已提交
1726 1727
	seq_printf(seq, "\nStopped: ");

1728
	list_for_each_entry(pkt_dev, &t->if_list, list)
L
Luiz Capitulino 已提交
1729
		if (!pkt_dev->running)
E
Eric Dumazet 已提交
1730
			seq_printf(seq, "%s ", pkt_dev->odevname);
L
Linus Torvalds 已提交
1731 1732

	if (t->result[0])
1733
		seq_printf(seq, "\nResult: %s\n", t->result);
L
Linus Torvalds 已提交
1734
	else
1735
		seq_printf(seq, "\nResult: NA\n");
L
Linus Torvalds 已提交
1736

L
Luiz Capitulino 已提交
1737
	if_unlock(t);
L
Linus Torvalds 已提交
1738

1739
	return 0;
L
Linus Torvalds 已提交
1740 1741
}

1742
static ssize_t pktgen_thread_write(struct file *file,
L
Luiz Capitulino 已提交
1743 1744
				   const char __user * user_buffer,
				   size_t count, loff_t * offset)
L
Linus Torvalds 已提交
1745
{
1746
	struct seq_file *seq = file->private_data;
L
Luiz Capitulino 已提交
1747
	struct pktgen_thread *t = seq->private;
1748
	int i, max, len, ret;
L
Linus Torvalds 已提交
1749
	char name[40];
L
Luiz Capitulino 已提交
1750
	char *pg_result;
1751

L
Linus Torvalds 已提交
1752
	if (count < 1) {
L
Luiz Capitulino 已提交
1753
		//      sprintf(pg_result, "Wrong command format");
L
Linus Torvalds 已提交
1754 1755
		return -EINVAL;
	}
1756

1757 1758
	max = count;
	len = count_trail_chars(user_buffer, max);
L
Luiz Capitulino 已提交
1759
	if (len < 0)
1760 1761
		return len;

1762
	i = len;
1763

L
Linus Torvalds 已提交
1764 1765 1766
	/* Read variable name */

	len = strn_len(&user_buffer[i], sizeof(name) - 1);
L
Luiz Capitulino 已提交
1767
	if (len < 0)
1768
		return len;
L
Luiz Capitulino 已提交
1769

L
Linus Torvalds 已提交
1770 1771 1772 1773
	memset(name, 0, sizeof(name));
	if (copy_from_user(name, &user_buffer[i], len))
		return -EFAULT;
	i += len;
1774

L
Luiz Capitulino 已提交
1775
	max = count - i;
L
Linus Torvalds 已提交
1776
	len = count_trail_chars(&user_buffer[i], max);
L
Luiz Capitulino 已提交
1777
	if (len < 0)
1778 1779
		return len;

L
Linus Torvalds 已提交
1780 1781
	i += len;

1782
	if (debug)
1783 1784
		printk(KERN_DEBUG "pktgen: t=%s, count=%lu\n",
		       name, (unsigned long)count);
L
Linus Torvalds 已提交
1785

L
Luiz Capitulino 已提交
1786
	if (!t) {
J
Joe Perches 已提交
1787
		pr_err("ERROR: No thread\n");
L
Linus Torvalds 已提交
1788 1789 1790 1791 1792 1793
		ret = -EINVAL;
		goto out;
	}

	pg_result = &(t->result[0]);

L
Luiz Capitulino 已提交
1794 1795 1796
	if (!strcmp(name, "add_device")) {
		char f[32];
		memset(f, 0, 32);
L
Linus Torvalds 已提交
1797
		len = strn_len(&user_buffer[i], sizeof(f) - 1);
L
Luiz Capitulino 已提交
1798 1799
		if (len < 0) {
			ret = len;
L
Linus Torvalds 已提交
1800 1801
			goto out;
		}
L
Luiz Capitulino 已提交
1802
		if (copy_from_user(f, &user_buffer[i], len))
L
Linus Torvalds 已提交
1803 1804
			return -EFAULT;
		i += len;
1805
		mutex_lock(&pktgen_thread_lock);
L
Luiz Capitulino 已提交
1806
		pktgen_add_device(t, f);
1807
		mutex_unlock(&pktgen_thread_lock);
L
Luiz Capitulino 已提交
1808 1809
		ret = count;
		sprintf(pg_result, "OK: add_device=%s", f);
L
Linus Torvalds 已提交
1810 1811 1812
		goto out;
	}

L
Luiz Capitulino 已提交
1813
	if (!strcmp(name, "rem_device_all")) {
1814
		mutex_lock(&pktgen_thread_lock);
1815
		t->control |= T_REMDEVALL;
1816
		mutex_unlock(&pktgen_thread_lock);
L
Luiz Capitulino 已提交
1817
		schedule_timeout_interruptible(msecs_to_jiffies(125));	/* Propagate thread->control  */
L
Linus Torvalds 已提交
1818
		ret = count;
L
Luiz Capitulino 已提交
1819
		sprintf(pg_result, "OK: rem_device_all");
L
Linus Torvalds 已提交
1820 1821 1822
		goto out;
	}

L
Luiz Capitulino 已提交
1823
	if (!strcmp(name, "max_before_softirq")) {
1824
		sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use");
L
Luiz Capitulino 已提交
1825
		ret = count;
L
Linus Torvalds 已提交
1826 1827 1828 1829
		goto out;
	}

	ret = -EINVAL;
L
Luiz Capitulino 已提交
1830
out:
L
Linus Torvalds 已提交
1831 1832 1833
	return ret;
}

1834
static int pktgen_thread_open(struct inode *inode, struct file *file)
L
Linus Torvalds 已提交
1835
{
1836
	return single_open(file, pktgen_thread_show, PDE(inode)->data);
L
Linus Torvalds 已提交
1837 1838
}

1839
static const struct file_operations pktgen_thread_fops = {
L
Luiz Capitulino 已提交
1840 1841 1842 1843 1844 1845
	.owner   = THIS_MODULE,
	.open    = pktgen_thread_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.write   = pktgen_thread_write,
	.release = single_release,
1846
};
L
Linus Torvalds 已提交
1847 1848

/* Think find or remove for NN */
L
Luiz Capitulino 已提交
1849
static struct pktgen_dev *__pktgen_NN_threads(const char *ifname, int remove)
L
Linus Torvalds 已提交
1850 1851 1852
{
	struct pktgen_thread *t;
	struct pktgen_dev *pkt_dev = NULL;
E
Eric Dumazet 已提交
1853
	bool exact = (remove == FIND);
L
Linus Torvalds 已提交
1854

1855
	list_for_each_entry(t, &pktgen_threads, th_list) {
E
Eric Dumazet 已提交
1856
		pkt_dev = pktgen_find_dev(t, ifname, exact);
L
Linus Torvalds 已提交
1857
		if (pkt_dev) {
L
Luiz Capitulino 已提交
1858 1859 1860 1861 1862 1863
			if (remove) {
				if_lock(t);
				pkt_dev->removal_mark = 1;
				t->control |= T_REMDEV;
				if_unlock(t);
			}
L
Linus Torvalds 已提交
1864 1865 1866
			break;
		}
	}
L
Luiz Capitulino 已提交
1867
	return pkt_dev;
L
Linus Torvalds 已提交
1868 1869
}

1870 1871 1872
/*
 * mark a device for removal
 */
1873
static void pktgen_mark_device(const char *ifname)
L
Linus Torvalds 已提交
1874 1875
{
	struct pktgen_dev *pkt_dev = NULL;
1876 1877 1878
	const int max_tries = 10, msec_per_try = 125;
	int i = 0;

1879
	mutex_lock(&pktgen_thread_lock);
J
Joe Perches 已提交
1880
	pr_debug("%s: marking %s for removal\n", __func__, ifname);
1881

L
Luiz Capitulino 已提交
1882
	while (1) {
1883 1884

		pkt_dev = __pktgen_NN_threads(ifname, REMOVE);
L
Luiz Capitulino 已提交
1885 1886
		if (pkt_dev == NULL)
			break;	/* success */
1887

1888
		mutex_unlock(&pktgen_thread_lock);
J
Joe Perches 已提交
1889 1890
		pr_debug("%s: waiting for %s to disappear....\n",
			 __func__, ifname);
1891
		schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
1892
		mutex_lock(&pktgen_thread_lock);
1893 1894

		if (++i >= max_tries) {
J
Joe Perches 已提交
1895 1896
			pr_err("%s: timed out after waiting %d msec for device %s to be removed\n",
			       __func__, msec_per_try * i, ifname);
1897 1898 1899 1900 1901
			break;
		}

	}

1902
	mutex_unlock(&pktgen_thread_lock);
1903
}
1904

1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
static void pktgen_change_name(struct net_device *dev)
{
	struct pktgen_thread *t;

	list_for_each_entry(t, &pktgen_threads, th_list) {
		struct pktgen_dev *pkt_dev;

		list_for_each_entry(pkt_dev, &t->if_list, list) {
			if (pkt_dev->odev != dev)
				continue;

			remove_proc_entry(pkt_dev->entry->name, pg_proc_dir);

A
Alexey Dobriyan 已提交
1918 1919 1920 1921
			pkt_dev->entry = proc_create_data(dev->name, 0600,
							  pg_proc_dir,
							  &pktgen_if_fops,
							  pkt_dev);
1922
			if (!pkt_dev->entry)
J
Joe Perches 已提交
1923 1924
				pr_err("can't move proc entry for '%s'\n",
				       dev->name);
1925 1926 1927
			break;
		}
	}
L
Linus Torvalds 已提交
1928 1929
}

L
Luiz Capitulino 已提交
1930 1931
static int pktgen_device_event(struct notifier_block *unused,
			       unsigned long event, void *ptr)
L
Linus Torvalds 已提交
1932
{
1933
	struct net_device *dev = ptr;
L
Linus Torvalds 已提交
1934

1935
	if (!net_eq(dev_net(dev), &init_net))
1936 1937
		return NOTIFY_DONE;

L
Linus Torvalds 已提交
1938 1939 1940 1941 1942
	/* It is OK that we do not hold the group lock right now,
	 * as we run under the RTNL lock.
	 */

	switch (event) {
1943 1944
	case NETDEV_CHANGENAME:
		pktgen_change_name(dev);
L
Linus Torvalds 已提交
1945
		break;
L
Luiz Capitulino 已提交
1946

L
Linus Torvalds 已提交
1947
	case NETDEV_UNREGISTER:
L
Luiz Capitulino 已提交
1948
		pktgen_mark_device(dev->name);
L
Linus Torvalds 已提交
1949
		break;
1950
	}
L
Linus Torvalds 已提交
1951 1952 1953 1954

	return NOTIFY_DONE;
}

1955 1956
static struct net_device *pktgen_dev_get_by_name(struct pktgen_dev *pkt_dev,
						 const char *ifname)
R
Robert Olsson 已提交
1957 1958
{
	char b[IFNAMSIZ+5];
1959
	int i;
R
Robert Olsson 已提交
1960

1961 1962
	for (i = 0; ifname[i] != '@'; i++) {
		if (i == IFNAMSIZ)
R
Robert Olsson 已提交
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
			break;

		b[i] = ifname[i];
	}
	b[i] = 0;

	return dev_get_by_name(&init_net, b);
}


L
Linus Torvalds 已提交
1973 1974
/* Associate pktgen_dev with a device. */

1975
static int pktgen_setup_dev(struct pktgen_dev *pkt_dev, const char *ifname)
L
Luiz Capitulino 已提交
1976
{
L
Linus Torvalds 已提交
1977
	struct net_device *odev;
1978
	int err;
L
Linus Torvalds 已提交
1979 1980 1981 1982

	/* Clean old setups */
	if (pkt_dev->odev) {
		dev_put(pkt_dev->odev);
L
Luiz Capitulino 已提交
1983 1984
		pkt_dev->odev = NULL;
	}
L
Linus Torvalds 已提交
1985

R
Robert Olsson 已提交
1986
	odev = pktgen_dev_get_by_name(pkt_dev, ifname);
L
Linus Torvalds 已提交
1987
	if (!odev) {
J
Joe Perches 已提交
1988
		pr_err("no such netdevice: \"%s\"\n", ifname);
1989
		return -ENODEV;
L
Linus Torvalds 已提交
1990
	}
1991

L
Linus Torvalds 已提交
1992
	if (odev->type != ARPHRD_ETHER) {
J
Joe Perches 已提交
1993
		pr_err("not an ethernet device: \"%s\"\n", ifname);
1994 1995
		err = -EINVAL;
	} else if (!netif_running(odev)) {
J
Joe Perches 已提交
1996
		pr_err("device is down: \"%s\"\n", ifname);
1997 1998 1999 2000
		err = -ENETDOWN;
	} else {
		pkt_dev->odev = odev;
		return 0;
L
Linus Torvalds 已提交
2001 2002 2003
	}

	dev_put(odev);
2004
	return err;
L
Linus Torvalds 已提交
2005 2006 2007 2008 2009 2010 2011
}

/* Read pkt_dev from the interface and set up internal pktgen_dev
 * structure to have the right information to create/send packets
 */
static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
{
2012 2013
	int ntxq;

L
Luiz Capitulino 已提交
2014
	if (!pkt_dev->odev) {
J
Joe Perches 已提交
2015
		pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n");
L
Luiz Capitulino 已提交
2016 2017 2018 2019 2020
		sprintf(pkt_dev->result,
			"ERROR: pkt_dev->odev == NULL in setup_inject.\n");
		return;
	}

2021 2022
	/* make sure that we don't pick a non-existing transmit queue */
	ntxq = pkt_dev->odev->real_num_tx_queues;
R
Robert Olsson 已提交
2023

2024
	if (ntxq <= pkt_dev->queue_map_min) {
J
Joe Perches 已提交
2025 2026 2027
		pr_warning("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
			   pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq,
			   pkt_dev->odevname);
2028 2029
		pkt_dev->queue_map_min = ntxq - 1;
	}
2030
	if (pkt_dev->queue_map_max >= ntxq) {
J
Joe Perches 已提交
2031 2032 2033
		pr_warning("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
			   pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq,
			   pkt_dev->odevname);
2034 2035 2036
		pkt_dev->queue_map_max = ntxq - 1;
	}

L
Luiz Capitulino 已提交
2037
	/* Default to the interface's mac if not explicitly set. */
L
Linus Torvalds 已提交
2038

2039
	if (is_zero_ether_addr(pkt_dev->src_mac))
L
Luiz Capitulino 已提交
2040
		memcpy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr, ETH_ALEN);
L
Linus Torvalds 已提交
2041

L
Luiz Capitulino 已提交
2042
	/* Set up Dest MAC */
2043
	memcpy(&(pkt_dev->hh[0]), pkt_dev->dst_mac, ETH_ALEN);
L
Linus Torvalds 已提交
2044

L
Luiz Capitulino 已提交
2045 2046 2047 2048
	/* Set up pkt size */
	pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size;

	if (pkt_dev->flags & F_IPV6) {
L
Linus Torvalds 已提交
2049
		/*
2050
		 * Skip this automatic address setting until locks or functions
L
Linus Torvalds 已提交
2051 2052 2053 2054
		 * gets exported
		 */

#ifdef NOTNOW
L
Luiz Capitulino 已提交
2055
		int i, set = 0, err = 1;
L
Linus Torvalds 已提交
2056 2057
		struct inet6_dev *idev;

L
Luiz Capitulino 已提交
2058 2059
		for (i = 0; i < IN6_ADDR_HSIZE; i++)
			if (pkt_dev->cur_in6_saddr.s6_addr[i]) {
L
Linus Torvalds 已提交
2060 2061 2062 2063
				set = 1;
				break;
			}

L
Luiz Capitulino 已提交
2064 2065
		if (!set) {

L
Linus Torvalds 已提交
2066 2067 2068 2069 2070 2071
			/*
			 * Use linklevel address if unconfigured.
			 *
			 * use ipv6_get_lladdr if/when it's get exported
			 */

2072
			rcu_read_lock();
2073 2074
			idev = __in6_dev_get(pkt_dev->odev);
			if (idev) {
L
Linus Torvalds 已提交
2075 2076 2077
				struct inet6_ifaddr *ifp;

				read_lock_bh(&idev->lock);
L
Luiz Capitulino 已提交
2078 2079
				for (ifp = idev->addr_list; ifp;
				     ifp = ifp->if_next) {
2080 2081
					if (ifp->scope == IFA_LINK &&
					    !(ifp->flags & IFA_F_TENTATIVE)) {
L
Luiz Capitulino 已提交
2082 2083 2084
						ipv6_addr_copy(&pkt_dev->
							       cur_in6_saddr,
							       &ifp->addr);
L
Linus Torvalds 已提交
2085 2086 2087 2088 2089 2090
						err = 0;
						break;
					}
				}
				read_unlock_bh(&idev->lock);
			}
2091
			rcu_read_unlock();
L
Luiz Capitulino 已提交
2092
			if (err)
J
Joe Perches 已提交
2093
				pr_err("ERROR: IPv6 link address not available\n");
L
Linus Torvalds 已提交
2094 2095
		}
#endif
L
Luiz Capitulino 已提交
2096
	} else {
L
Linus Torvalds 已提交
2097 2098 2099
		pkt_dev->saddr_min = 0;
		pkt_dev->saddr_max = 0;
		if (strlen(pkt_dev->src_min) == 0) {
L
Luiz Capitulino 已提交
2100 2101

			struct in_device *in_dev;
L
Linus Torvalds 已提交
2102 2103

			rcu_read_lock();
2104
			in_dev = __in_dev_get_rcu(pkt_dev->odev);
L
Linus Torvalds 已提交
2105 2106
			if (in_dev) {
				if (in_dev->ifa_list) {
L
Luiz Capitulino 已提交
2107 2108
					pkt_dev->saddr_min =
					    in_dev->ifa_list->ifa_address;
L
Linus Torvalds 已提交
2109 2110 2111 2112
					pkt_dev->saddr_max = pkt_dev->saddr_min;
				}
			}
			rcu_read_unlock();
L
Luiz Capitulino 已提交
2113
		} else {
L
Linus Torvalds 已提交
2114 2115 2116 2117 2118 2119 2120
			pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
			pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
		}

		pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
		pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
	}
L
Luiz Capitulino 已提交
2121 2122 2123 2124 2125 2126 2127
	/* Initialize current values. */
	pkt_dev->cur_dst_mac_offset = 0;
	pkt_dev->cur_src_mac_offset = 0;
	pkt_dev->cur_saddr = pkt_dev->saddr_min;
	pkt_dev->cur_daddr = pkt_dev->daddr_min;
	pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
	pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
L
Linus Torvalds 已提交
2128 2129 2130
	pkt_dev->nflows = 0;
}

2131 2132 2133

static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
{
2134
	ktime_t start_time, end_time;
E
Eric Dumazet 已提交
2135
	s64 remaining;
S
Stephen Hemminger 已提交
2136 2137 2138 2139 2140
	struct hrtimer_sleeper t;

	hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
	hrtimer_set_expires(&t.timer, spin_until);

2141
	remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
E
Eric Dumazet 已提交
2142 2143
	if (remaining <= 0) {
		pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay);
S
Stephen Hemminger 已提交
2144
		return;
E
Eric Dumazet 已提交
2145
	}
S
Stephen Hemminger 已提交
2146

2147
	start_time = ktime_now();
2148 2149
	if (remaining < 100000)
		ndelay(remaining);	/* really small just spin */
S
Stephen Hemminger 已提交
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
	else {
		/* see do_nanosleep */
		hrtimer_init_sleeper(&t, current);
		do {
			set_current_state(TASK_INTERRUPTIBLE);
			hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS);
			if (!hrtimer_active(&t.timer))
				t.task = NULL;

			if (likely(t.task))
L
Linus Torvalds 已提交
2160 2161
				schedule();

S
Stephen Hemminger 已提交
2162 2163 2164
			hrtimer_cancel(&t.timer);
		} while (t.task && pkt_dev->running && !signal_pending(current));
		__set_current_state(TASK_RUNNING);
L
Linus Torvalds 已提交
2165
	}
2166 2167 2168
	end_time = ktime_now();

	pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time));
2169
	pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay);
L
Linus Torvalds 已提交
2170 2171
}

2172 2173
static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev)
{
J
Jamal Hadi Salim 已提交
2174
	pkt_dev->pkt_overhead = 0;
2175 2176 2177 2178 2179
	pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32);
	pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev);
	pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev);
}

S
Stephen Hemminger 已提交
2180
static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow)
2181
{
S
Stephen Hemminger 已提交
2182
	return !!(pkt_dev->flows[flow].flags & F_INIT);
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
}

static inline int f_pick(struct pktgen_dev *pkt_dev)
{
	int flow = pkt_dev->curfl;

	if (pkt_dev->flags & F_FLOW_SEQ) {
		if (pkt_dev->flows[flow].count >= pkt_dev->lflow) {
			/* reset time */
			pkt_dev->flows[flow].count = 0;
R
Robert Olsson 已提交
2193
			pkt_dev->flows[flow].flags = 0;
2194 2195 2196 2197 2198 2199
			pkt_dev->curfl += 1;
			if (pkt_dev->curfl >= pkt_dev->cflows)
				pkt_dev->curfl = 0; /*reset */
		}
	} else {
		flow = random32() % pkt_dev->cflows;
R
Robert Olsson 已提交
2200
		pkt_dev->curfl = flow;
2201

R
Robert Olsson 已提交
2202
		if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
2203
			pkt_dev->flows[flow].count = 0;
R
Robert Olsson 已提交
2204 2205
			pkt_dev->flows[flow].flags = 0;
		}
2206 2207 2208 2209 2210
	}

	return pkt_dev->curfl;
}

J
Jamal Hadi Salim 已提交
2211 2212 2213 2214 2215

#ifdef CONFIG_XFRM
/* If there was already an IPSEC SA, we keep it as is, else
 * we go look for it ...
*/
2216
#define DUMMY_MARK 0
2217
static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow)
J
Jamal Hadi Salim 已提交
2218 2219 2220 2221
{
	struct xfrm_state *x = pkt_dev->flows[flow].x;
	if (!x) {
		/*slow path: we dont already have xfrm_state*/
2222
		x = xfrm_stateonly_find(&init_net, DUMMY_MARK,
2223
					(xfrm_address_t *)&pkt_dev->cur_daddr,
J
Jamal Hadi Salim 已提交
2224 2225 2226 2227 2228 2229 2230
					(xfrm_address_t *)&pkt_dev->cur_saddr,
					AF_INET,
					pkt_dev->ipsmode,
					pkt_dev->ipsproto, 0);
		if (x) {
			pkt_dev->flows[flow].x = x;
			set_pkt_overhead(pkt_dev);
2231
			pkt_dev->pkt_overhead += x->props.header_len;
J
Jamal Hadi Salim 已提交
2232 2233 2234 2235 2236
		}

	}
}
#endif
2237 2238
static void set_cur_queue_map(struct pktgen_dev *pkt_dev)
{
R
Robert Olsson 已提交
2239 2240 2241 2242

	if (pkt_dev->flags & F_QUEUE_MAP_CPU)
		pkt_dev->cur_queue_map = smp_processor_id();

E
Eric Dumazet 已提交
2243
	else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) {
2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
		__u16 t;
		if (pkt_dev->flags & F_QUEUE_MAP_RND) {
			t = random32() %
				(pkt_dev->queue_map_max -
				 pkt_dev->queue_map_min + 1)
				+ pkt_dev->queue_map_min;
		} else {
			t = pkt_dev->cur_queue_map + 1;
			if (t > pkt_dev->queue_map_max)
				t = pkt_dev->queue_map_min;
		}
		pkt_dev->cur_queue_map = t;
	}
R
Robert Olsson 已提交
2257
	pkt_dev->cur_queue_map  = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues;
2258 2259
}

L
Linus Torvalds 已提交
2260 2261 2262
/* Increment/randomize headers according to flags and current values
 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst
 */
L
Luiz Capitulino 已提交
2263 2264 2265 2266 2267
static void mod_cur_headers(struct pktgen_dev *pkt_dev)
{
	__u32 imn;
	__u32 imx;
	int flow = 0;
L
Linus Torvalds 已提交
2268

2269 2270
	if (pkt_dev->cflows)
		flow = f_pick(pkt_dev);
L
Linus Torvalds 已提交
2271 2272

	/*  Deal with source MAC */
L
Luiz Capitulino 已提交
2273 2274 2275 2276 2277
	if (pkt_dev->src_mac_count > 1) {
		__u32 mc;
		__u32 tmp;

		if (pkt_dev->flags & F_MACSRC_RND)
S
Stephen Hemminger 已提交
2278
			mc = random32() % pkt_dev->src_mac_count;
L
Luiz Capitulino 已提交
2279 2280
		else {
			mc = pkt_dev->cur_src_mac_offset++;
R
Robert Olsson 已提交
2281
			if (pkt_dev->cur_src_mac_offset >=
L
Luiz Capitulino 已提交
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303
			    pkt_dev->src_mac_count)
				pkt_dev->cur_src_mac_offset = 0;
		}

		tmp = pkt_dev->src_mac[5] + (mc & 0xFF);
		pkt_dev->hh[11] = tmp;
		tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[10] = tmp;
		tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[9] = tmp;
		tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[8] = tmp;
		tmp = (pkt_dev->src_mac[1] + (tmp >> 8));
		pkt_dev->hh[7] = tmp;
	}

	/*  Deal with Destination MAC */
	if (pkt_dev->dst_mac_count > 1) {
		__u32 mc;
		__u32 tmp;

		if (pkt_dev->flags & F_MACDST_RND)
S
Stephen Hemminger 已提交
2304
			mc = random32() % pkt_dev->dst_mac_count;
L
Luiz Capitulino 已提交
2305 2306 2307

		else {
			mc = pkt_dev->cur_dst_mac_offset++;
R
Robert Olsson 已提交
2308
			if (pkt_dev->cur_dst_mac_offset >=
L
Luiz Capitulino 已提交
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
			    pkt_dev->dst_mac_count) {
				pkt_dev->cur_dst_mac_offset = 0;
			}
		}

		tmp = pkt_dev->dst_mac[5] + (mc & 0xFF);
		pkt_dev->hh[5] = tmp;
		tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[4] = tmp;
		tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[3] = tmp;
		tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
		pkt_dev->hh[2] = tmp;
		tmp = (pkt_dev->dst_mac[1] + (tmp >> 8));
		pkt_dev->hh[1] = tmp;
	}

2326 2327
	if (pkt_dev->flags & F_MPLS_RND) {
		unsigned i;
2328
		for (i = 0; i < pkt_dev->nr_labels; i++)
2329 2330
			if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM)
				pkt_dev->labels[i] = MPLS_STACK_BOTTOM |
S
Stephen Hemminger 已提交
2331
					     ((__force __be32)random32() &
2332 2333 2334
						      htonl(0x000fffff));
	}

F
Francesco Fondelli 已提交
2335
	if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) {
S
Stephen Hemminger 已提交
2336
		pkt_dev->vlan_id = random32() & (4096-1);
F
Francesco Fondelli 已提交
2337 2338 2339
	}

	if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) {
S
Stephen Hemminger 已提交
2340
		pkt_dev->svlan_id = random32() & (4096 - 1);
F
Francesco Fondelli 已提交
2341 2342
	}

L
Luiz Capitulino 已提交
2343 2344
	if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) {
		if (pkt_dev->flags & F_UDPSRC_RND)
S
Stephen Hemminger 已提交
2345 2346 2347
			pkt_dev->cur_udp_src = random32() %
				(pkt_dev->udp_src_max - pkt_dev->udp_src_min)
				+ pkt_dev->udp_src_min;
L
Luiz Capitulino 已提交
2348 2349

		else {
L
Linus Torvalds 已提交
2350 2351 2352
			pkt_dev->cur_udp_src++;
			if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max)
				pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
L
Luiz Capitulino 已提交
2353 2354 2355 2356 2357
		}
	}

	if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) {
		if (pkt_dev->flags & F_UDPDST_RND) {
S
Stephen Hemminger 已提交
2358 2359 2360
			pkt_dev->cur_udp_dst = random32() %
				(pkt_dev->udp_dst_max - pkt_dev->udp_dst_min)
				+ pkt_dev->udp_dst_min;
L
Luiz Capitulino 已提交
2361
		} else {
L
Linus Torvalds 已提交
2362
			pkt_dev->cur_udp_dst++;
L
Luiz Capitulino 已提交
2363
			if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max)
L
Linus Torvalds 已提交
2364
				pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
L
Luiz Capitulino 已提交
2365 2366
		}
	}
L
Linus Torvalds 已提交
2367 2368 2369

	if (!(pkt_dev->flags & F_IPV6)) {

2370 2371 2372
		imn = ntohl(pkt_dev->saddr_min);
		imx = ntohl(pkt_dev->saddr_max);
		if (imn < imx) {
L
Linus Torvalds 已提交
2373
			__u32 t;
L
Luiz Capitulino 已提交
2374
			if (pkt_dev->flags & F_IPSRC_RND)
S
Stephen Hemminger 已提交
2375
				t = random32() % (imx - imn) + imn;
L
Linus Torvalds 已提交
2376 2377 2378
			else {
				t = ntohl(pkt_dev->cur_saddr);
				t++;
2379
				if (t > imx)
L
Linus Torvalds 已提交
2380
					t = imn;
2381

L
Linus Torvalds 已提交
2382 2383 2384
			}
			pkt_dev->cur_saddr = htonl(t);
		}
L
Luiz Capitulino 已提交
2385

2386
		if (pkt_dev->cflows && f_seen(pkt_dev, flow)) {
L
Linus Torvalds 已提交
2387 2388
			pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr;
		} else {
A
Al Viro 已提交
2389 2390 2391
			imn = ntohl(pkt_dev->daddr_min);
			imx = ntohl(pkt_dev->daddr_max);
			if (imn < imx) {
L
Linus Torvalds 已提交
2392
				__u32 t;
A
Al Viro 已提交
2393
				__be32 s;
L
Linus Torvalds 已提交
2394 2395
				if (pkt_dev->flags & F_IPDST_RND) {

S
Stephen Hemminger 已提交
2396
					t = random32() % (imx - imn) + imn;
A
Al Viro 已提交
2397
					s = htonl(t);
L
Linus Torvalds 已提交
2398

2399 2400
					while (ipv4_is_loopback(s) ||
					       ipv4_is_multicast(s) ||
2401
					       ipv4_is_lbcast(s) ||
2402 2403
					       ipv4_is_zeronet(s) ||
					       ipv4_is_local_multicast(s)) {
S
Stephen Hemminger 已提交
2404
						t = random32() % (imx - imn) + imn;
A
Al Viro 已提交
2405
						s = htonl(t);
L
Linus Torvalds 已提交
2406
					}
A
Al Viro 已提交
2407 2408
					pkt_dev->cur_daddr = s;
				} else {
L
Linus Torvalds 已提交
2409 2410 2411 2412 2413 2414 2415 2416
					t = ntohl(pkt_dev->cur_daddr);
					t++;
					if (t > imx) {
						t = imn;
					}
					pkt_dev->cur_daddr = htonl(t);
				}
			}
L
Luiz Capitulino 已提交
2417
			if (pkt_dev->cflows) {
2418
				pkt_dev->flows[flow].flags |= F_INIT;
L
Luiz Capitulino 已提交
2419 2420
				pkt_dev->flows[flow].cur_daddr =
				    pkt_dev->cur_daddr;
J
Jamal Hadi Salim 已提交
2421 2422 2423 2424
#ifdef CONFIG_XFRM
				if (pkt_dev->flags & F_IPSEC_ON)
					get_ipsec_sa(pkt_dev, flow);
#endif
L
Linus Torvalds 已提交
2425 2426 2427
				pkt_dev->nflows++;
			}
		}
L
Luiz Capitulino 已提交
2428 2429 2430 2431 2432 2433
	} else {		/* IPV6 * */

		if (pkt_dev->min_in6_daddr.s6_addr32[0] == 0 &&
		    pkt_dev->min_in6_daddr.s6_addr32[1] == 0 &&
		    pkt_dev->min_in6_daddr.s6_addr32[2] == 0 &&
		    pkt_dev->min_in6_daddr.s6_addr32[3] == 0) ;
L
Linus Torvalds 已提交
2434 2435 2436 2437 2438
		else {
			int i;

			/* Only random destinations yet */

L
Luiz Capitulino 已提交
2439
			for (i = 0; i < 4; i++) {
L
Linus Torvalds 已提交
2440
				pkt_dev->cur_in6_daddr.s6_addr32[i] =
S
Stephen Hemminger 已提交
2441
				    (((__force __be32)random32() |
L
Luiz Capitulino 已提交
2442 2443
				      pkt_dev->min_in6_daddr.s6_addr32[i]) &
				     pkt_dev->max_in6_daddr.s6_addr32[i]);
L
Linus Torvalds 已提交
2444
			}
L
Luiz Capitulino 已提交
2445
		}
L
Linus Torvalds 已提交
2446 2447
	}

L
Luiz Capitulino 已提交
2448 2449 2450
	if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) {
		__u32 t;
		if (pkt_dev->flags & F_TXSIZE_RND) {
S
Stephen Hemminger 已提交
2451 2452 2453
			t = random32() %
				(pkt_dev->max_pkt_size - pkt_dev->min_pkt_size)
				+ pkt_dev->min_pkt_size;
L
Luiz Capitulino 已提交
2454
		} else {
L
Linus Torvalds 已提交
2455
			t = pkt_dev->cur_pkt_size + 1;
L
Luiz Capitulino 已提交
2456
			if (t > pkt_dev->max_pkt_size)
L
Linus Torvalds 已提交
2457
				t = pkt_dev->min_pkt_size;
L
Luiz Capitulino 已提交
2458 2459 2460
		}
		pkt_dev->cur_pkt_size = t;
	}
L
Linus Torvalds 已提交
2461

2462
	set_cur_queue_map(pkt_dev);
R
Robert Olsson 已提交
2463

L
Linus Torvalds 已提交
2464 2465 2466
	pkt_dev->flows[flow].count++;
}

J
Jamal Hadi Salim 已提交
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482

#ifdef CONFIG_XFRM
static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev)
{
	struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
	int err = 0;

	if (!x)
		return 0;
	/* XXX: we dont support tunnel mode for now until
	 * we resolve the dst issue */
	if (x->props.mode != XFRM_MODE_TRANSPORT)
		return 0;

	spin_lock(&x->lock);

2483
	err = x->outer_mode->output(x, skb);
J
Jamal Hadi Salim 已提交
2484 2485 2486 2487 2488 2489
	if (err)
		goto error;
	err = x->type->output(x, skb);
	if (err)
		goto error;

2490
	x->curlft.bytes += skb->len;
J
Jamal Hadi Salim 已提交
2491 2492 2493 2494 2495 2496
	x->curlft.packets++;
error:
	spin_unlock(&x->lock);
	return err;
}

S
Stephen Hemminger 已提交
2497
static void free_SAs(struct pktgen_dev *pkt_dev)
J
Jamal Hadi Salim 已提交
2498 2499 2500
{
	if (pkt_dev->cflows) {
		/* let go of the SAs if we have them */
2501 2502
		int i;
		for (i = 0; i < pkt_dev->cflows; i++) {
J
Jamal Hadi Salim 已提交
2503 2504 2505 2506 2507 2508 2509 2510 2511
			struct xfrm_state *x = pkt_dev->flows[i].x;
			if (x) {
				xfrm_state_put(x);
				pkt_dev->flows[i].x = NULL;
			}
		}
	}
}

S
Stephen Hemminger 已提交
2512
static int process_ipsec(struct pktgen_dev *pkt_dev,
J
Jamal Hadi Salim 已提交
2513 2514 2515 2516 2517 2518 2519 2520 2521
			      struct sk_buff *skb, __be16 protocol)
{
	if (pkt_dev->flags & F_IPSEC_ON) {
		struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
		int nhead = 0;
		if (x) {
			int ret;
			__u8 *eth;
			nhead = x->props.header_len - skb_headroom(skb);
2522
			if (nhead > 0) {
J
Jamal Hadi Salim 已提交
2523 2524
				ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC);
				if (ret < 0) {
J
Joe Perches 已提交
2525 2526
					pr_err("Error expanding ipsec packet %d\n",
					       ret);
2527
					goto err;
J
Jamal Hadi Salim 已提交
2528 2529 2530 2531 2532 2533 2534
				}
			}

			/* ipsec is not expecting ll header */
			skb_pull(skb, ETH_HLEN);
			ret = pktgen_output_ipsec(skb, pkt_dev);
			if (ret) {
J
Joe Perches 已提交
2535
				pr_err("Error creating ipsec packet %d\n", ret);
2536
				goto err;
J
Jamal Hadi Salim 已提交
2537 2538 2539 2540
			}
			/* restore ll */
			eth = (__u8 *) skb_push(skb, ETH_HLEN);
			memcpy(eth, pkt_dev->hh, 12);
2541
			*(u16 *) &eth[12] = protocol;
J
Jamal Hadi Salim 已提交
2542 2543 2544
		}
	}
	return 1;
2545 2546 2547
err:
	kfree_skb(skb);
	return 0;
J
Jamal Hadi Salim 已提交
2548 2549 2550
}
#endif

2551 2552 2553
static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev)
{
	unsigned i;
2554
	for (i = 0; i < pkt_dev->nr_labels; i++)
2555
		*mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM;
2556

2557 2558 2559 2560
	mpls--;
	*mpls |= MPLS_STACK_BOTTOM;
}

A
Al Viro 已提交
2561 2562 2563 2564 2565 2566
static inline __be16 build_tci(unsigned int id, unsigned int cfi,
			       unsigned int prio)
{
	return htons(id | (cfi << 12) | (prio << 13));
}

E
Eric Dumazet 已提交
2567 2568 2569 2570 2571 2572 2573 2574 2575 2576
static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb,
				int datalen)
{
	struct timeval timestamp;
	struct pktgen_hdr *pgh;

	pgh = (struct pktgen_hdr *)skb_put(skb, sizeof(*pgh));
	datalen -= sizeof(*pgh);

	if (pkt_dev->nfrags <= 0) {
2577
		memset(skb_put(skb, datalen), 0, datalen);
E
Eric Dumazet 已提交
2578 2579 2580
	} else {
		int frags = pkt_dev->nfrags;
		int i, len;
2581
		int frag_len;
E
Eric Dumazet 已提交
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592


		if (frags > MAX_SKB_FRAGS)
			frags = MAX_SKB_FRAGS;
		len = datalen - frags * PAGE_SIZE;
		if (len > 0) {
			memset(skb_put(skb, len), 0, len);
			datalen = frags * PAGE_SIZE;
		}

		i = 0;
2593 2594
		frag_len = (datalen/frags) < PAGE_SIZE ?
			   (datalen/frags) : PAGE_SIZE;
E
Eric Dumazet 已提交
2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
		while (datalen > 0) {
			if (unlikely(!pkt_dev->page)) {
				int node = numa_node_id();

				if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE))
					node = pkt_dev->node;
				pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
				if (!pkt_dev->page)
					break;
			}
			skb_shinfo(skb)->frags[i].page = pkt_dev->page;
			get_page(pkt_dev->page);
			skb_shinfo(skb)->frags[i].page_offset = 0;
2608 2609 2610 2611 2612 2613
			/*last fragment, fill rest of data*/
			if (i == (frags - 1))
				skb_shinfo(skb)->frags[i].size =
				    (datalen < PAGE_SIZE ? datalen : PAGE_SIZE);
			else
				skb_shinfo(skb)->frags[i].size = frag_len;
E
Eric Dumazet 已提交
2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632
			datalen -= skb_shinfo(skb)->frags[i].size;
			skb->len += skb_shinfo(skb)->frags[i].size;
			skb->data_len += skb_shinfo(skb)->frags[i].size;
			i++;
			skb_shinfo(skb)->nr_frags = i;
		}
	}

	/* Stamp the time, and sequence number,
	 * convert them to network byte order
	 */
	pgh->pgh_magic = htonl(PKTGEN_MAGIC);
	pgh->seq_num = htonl(pkt_dev->seq_num);

	do_gettimeofday(&timestamp);
	pgh->tv_sec = htonl(timestamp.tv_sec);
	pgh->tv_usec = htonl(timestamp.tv_usec);
}

L
Luiz Capitulino 已提交
2633 2634
static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
					struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
2635 2636 2637 2638 2639 2640
{
	struct sk_buff *skb = NULL;
	__u8 *eth;
	struct udphdr *udph;
	int datalen, iplen;
	struct iphdr *iph;
2641
	__be16 protocol = htons(ETH_P_IP);
2642
	__be32 *mpls;
F
Francesco Fondelli 已提交
2643 2644 2645 2646
	__be16 *vlan_tci = NULL;                 /* Encapsulates priority and VLAN ID */
	__be16 *vlan_encapsulated_proto = NULL;  /* packet type ID field (or len) for VLAN tag */
	__be16 *svlan_tci = NULL;                /* Encapsulates priority and SVLAN ID */
	__be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
2647
	u16 queue_map;
2648 2649

	if (pkt_dev->nr_labels)
2650
		protocol = htons(ETH_P_MPLS_UC);
L
Luiz Capitulino 已提交
2651

F
Francesco Fondelli 已提交
2652
	if (pkt_dev->vlan_id != 0xffff)
2653
		protocol = htons(ETH_P_8021Q);
F
Francesco Fondelli 已提交
2654

2655 2656 2657 2658
	/* Update any of the values, used when we're incrementing various
	 * fields.
	 */
	mod_cur_headers(pkt_dev);
2659
	queue_map = pkt_dev->cur_queue_map;
2660

2661
	datalen = (odev->hard_header_len + 16) & ~0xf;
R
Robert Olsson 已提交
2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682

	if (pkt_dev->flags & F_NODE) {
		int node;

		if (pkt_dev->node >= 0)
			node = pkt_dev->node;
		else
			node =  numa_node_id();

		skb = __alloc_skb(NET_SKB_PAD + pkt_dev->cur_pkt_size + 64
				  + datalen + pkt_dev->pkt_overhead, GFP_NOWAIT, 0, node);
		if (likely(skb)) {
			skb_reserve(skb, NET_SKB_PAD);
			skb->dev = odev;
		}
	}
	else
	  skb = __netdev_alloc_skb(odev,
				   pkt_dev->cur_pkt_size + 64
				   + datalen + pkt_dev->pkt_overhead, GFP_NOWAIT);

L
Linus Torvalds 已提交
2683 2684 2685 2686
	if (!skb) {
		sprintf(pkt_dev->result, "No memory");
		return NULL;
	}
J
Junchang Wang 已提交
2687
	prefetchw(skb->data);
L
Linus Torvalds 已提交
2688

2689
	skb_reserve(skb, datalen);
L
Linus Torvalds 已提交
2690 2691 2692

	/*  Reserve for ethernet and IP header  */
	eth = (__u8 *) skb_push(skb, 14);
2693 2694 2695
	mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32));
	if (pkt_dev->nr_labels)
		mpls_push(mpls, pkt_dev);
F
Francesco Fondelli 已提交
2696 2697

	if (pkt_dev->vlan_id != 0xffff) {
2698
		if (pkt_dev->svlan_id != 0xffff) {
F
Francesco Fondelli 已提交
2699
			svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16));
A
Al Viro 已提交
2700 2701 2702
			*svlan_tci = build_tci(pkt_dev->svlan_id,
					       pkt_dev->svlan_cfi,
					       pkt_dev->svlan_p);
F
Francesco Fondelli 已提交
2703
			svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16));
2704
			*svlan_encapsulated_proto = htons(ETH_P_8021Q);
F
Francesco Fondelli 已提交
2705 2706
		}
		vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16));
A
Al Viro 已提交
2707 2708 2709
		*vlan_tci = build_tci(pkt_dev->vlan_id,
				      pkt_dev->vlan_cfi,
				      pkt_dev->vlan_p);
F
Francesco Fondelli 已提交
2710
		vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16));
2711
		*vlan_encapsulated_proto = htons(ETH_P_IP);
F
Francesco Fondelli 已提交
2712 2713
	}

2714
	skb->network_header = skb->tail;
2715
	skb->transport_header = skb->network_header + sizeof(struct iphdr);
2716
	skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr));
2717
	skb_set_queue_mapping(skb, queue_map);
J
John Fastabend 已提交
2718 2719
	skb->priority = pkt_dev->skb_priority;

2720 2721
	iph = ip_hdr(skb);
	udph = udp_hdr(skb);
L
Linus Torvalds 已提交
2722 2723

	memcpy(eth, pkt_dev->hh, 12);
A
Al Viro 已提交
2724
	*(__be16 *) & eth[12] = protocol;
L
Linus Torvalds 已提交
2725

2726 2727
	/* Eth + IPh + UDPh + mpls */
	datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 -
2728
		  pkt_dev->pkt_overhead;
L
Luiz Capitulino 已提交
2729
	if (datalen < sizeof(struct pktgen_hdr))
L
Linus Torvalds 已提交
2730
		datalen = sizeof(struct pktgen_hdr);
L
Luiz Capitulino 已提交
2731

L
Linus Torvalds 已提交
2732 2733
	udph->source = htons(pkt_dev->cur_udp_src);
	udph->dest = htons(pkt_dev->cur_udp_dst);
L
Luiz Capitulino 已提交
2734 2735
	udph->len = htons(datalen + 8);	/* DATA + udphdr */
	udph->check = 0;	/* No checksum */
L
Linus Torvalds 已提交
2736 2737 2738 2739

	iph->ihl = 5;
	iph->version = 4;
	iph->ttl = 32;
F
Francesco Fondelli 已提交
2740
	iph->tos = pkt_dev->tos;
L
Luiz Capitulino 已提交
2741
	iph->protocol = IPPROTO_UDP;	/* UDP */
L
Linus Torvalds 已提交
2742 2743
	iph->saddr = pkt_dev->cur_saddr;
	iph->daddr = pkt_dev->cur_daddr;
E
Eric Dumazet 已提交
2744 2745
	iph->id = htons(pkt_dev->ip_id);
	pkt_dev->ip_id++;
L
Linus Torvalds 已提交
2746 2747 2748 2749
	iph->frag_off = 0;
	iplen = 20 + 8 + datalen;
	iph->tot_len = htons(iplen);
	iph->check = 0;
L
Luiz Capitulino 已提交
2750
	iph->check = ip_fast_csum((void *)iph, iph->ihl);
2751
	skb->protocol = protocol;
2752
	skb->mac_header = (skb->network_header - ETH_HLEN -
2753
			   pkt_dev->pkt_overhead);
L
Linus Torvalds 已提交
2754 2755
	skb->dev = odev;
	skb->pkt_type = PACKET_HOST;
E
Eric Dumazet 已提交
2756
	pktgen_finalize_skb(pkt_dev, skb, datalen);
L
Linus Torvalds 已提交
2757

J
Jamal Hadi Salim 已提交
2758 2759 2760 2761 2762
#ifdef CONFIG_XFRM
	if (!process_ipsec(pkt_dev, skb, protocol))
		return NULL;
#endif

L
Linus Torvalds 已提交
2763 2764 2765 2766
	return skb;
}

/*
2767
 * scan_ip6, fmt_ip taken from dietlibc-0.21
L
Linus Torvalds 已提交
2768 2769
 * Author Felix von Leitner <felix-dietlibc@fefe.de>
 *
2770
 * Slightly modified for kernel.
L
Linus Torvalds 已提交
2771 2772 2773 2774
 * Should be candidate for net/ipv4/utils.c
 * --ro
 */

L
Luiz Capitulino 已提交
2775
static unsigned int scan_ip6(const char *s, char ip[16])
L
Linus Torvalds 已提交
2776 2777
{
	unsigned int i;
L
Luiz Capitulino 已提交
2778
	unsigned int len = 0;
L
Linus Torvalds 已提交
2779 2780
	unsigned long u;
	char suffix[16];
L
Luiz Capitulino 已提交
2781 2782
	unsigned int prefixlen = 0;
	unsigned int suffixlen = 0;
A
Al Viro 已提交
2783
	__be32 tmp;
S
Stephen Hemminger 已提交
2784
	char *pos;
L
Linus Torvalds 已提交
2785

L
Luiz Capitulino 已提交
2786 2787
	for (i = 0; i < 16; i++)
		ip[i] = 0;
L
Linus Torvalds 已提交
2788 2789 2790 2791

	for (;;) {
		if (*s == ':') {
			len++;
L
Luiz Capitulino 已提交
2792 2793
			if (s[1] == ':') {	/* Found "::", skip to part 2 */
				s += 2;
L
Linus Torvalds 已提交
2794 2795 2796 2797 2798 2799
				len++;
				break;
			}
			s++;
		}

S
Stephen Hemminger 已提交
2800 2801
		u = simple_strtoul(s, &pos, 16);
		i = pos - s;
L
Luiz Capitulino 已提交
2802 2803 2804
		if (!i)
			return 0;
		if (prefixlen == 12 && s[i] == '.') {
L
Linus Torvalds 已提交
2805 2806 2807 2808

			/* the last 4 bytes may be written as IPv4 address */

			tmp = in_aton(s);
L
Luiz Capitulino 已提交
2809 2810
			memcpy((struct in_addr *)(ip + 12), &tmp, sizeof(tmp));
			return i + len;
L
Linus Torvalds 已提交
2811 2812 2813
		}
		ip[prefixlen++] = (u >> 8);
		ip[prefixlen++] = (u & 255);
L
Luiz Capitulino 已提交
2814 2815 2816
		s += i;
		len += i;
		if (prefixlen == 16)
L
Linus Torvalds 已提交
2817 2818 2819 2820 2821 2822
			return len;
	}

/* part 2, after "::" */
	for (;;) {
		if (*s == ':') {
L
Luiz Capitulino 已提交
2823
			if (suffixlen == 0)
L
Linus Torvalds 已提交
2824 2825 2826
				break;
			s++;
			len++;
L
Luiz Capitulino 已提交
2827
		} else if (suffixlen != 0)
L
Linus Torvalds 已提交
2828
			break;
S
Stephen Hemminger 已提交
2829 2830 2831

		u = simple_strtol(s, &pos, 16);
		i = pos - s;
L
Linus Torvalds 已提交
2832
		if (!i) {
L
Luiz Capitulino 已提交
2833 2834
			if (*s)
				len--;
L
Linus Torvalds 已提交
2835 2836
			break;
		}
L
Luiz Capitulino 已提交
2837
		if (suffixlen + prefixlen <= 12 && s[i] == '.') {
L
Linus Torvalds 已提交
2838
			tmp = in_aton(s);
L
Luiz Capitulino 已提交
2839 2840 2841 2842
			memcpy((struct in_addr *)(suffix + suffixlen), &tmp,
			       sizeof(tmp));
			suffixlen += 4;
			len += strlen(s);
L
Linus Torvalds 已提交
2843 2844 2845 2846
			break;
		}
		suffix[suffixlen++] = (u >> 8);
		suffix[suffixlen++] = (u & 255);
L
Luiz Capitulino 已提交
2847 2848 2849
		s += i;
		len += i;
		if (prefixlen + suffixlen == 16)
L
Linus Torvalds 已提交
2850 2851
			break;
	}
L
Luiz Capitulino 已提交
2852 2853
	for (i = 0; i < suffixlen; i++)
		ip[16 - suffixlen + i] = suffix[i];
L
Linus Torvalds 已提交
2854 2855 2856
	return len;
}

L
Luiz Capitulino 已提交
2857 2858
static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
					struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
2859 2860 2861 2862 2863 2864
{
	struct sk_buff *skb = NULL;
	__u8 *eth;
	struct udphdr *udph;
	int datalen;
	struct ipv6hdr *iph;
2865
	__be16 protocol = htons(ETH_P_IPV6);
2866
	__be32 *mpls;
F
Francesco Fondelli 已提交
2867 2868 2869 2870
	__be16 *vlan_tci = NULL;                 /* Encapsulates priority and VLAN ID */
	__be16 *vlan_encapsulated_proto = NULL;  /* packet type ID field (or len) for VLAN tag */
	__be16 *svlan_tci = NULL;                /* Encapsulates priority and SVLAN ID */
	__be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
2871
	u16 queue_map;
2872 2873

	if (pkt_dev->nr_labels)
2874
		protocol = htons(ETH_P_MPLS_UC);
2875

F
Francesco Fondelli 已提交
2876
	if (pkt_dev->vlan_id != 0xffff)
2877
		protocol = htons(ETH_P_8021Q);
F
Francesco Fondelli 已提交
2878

2879 2880 2881 2882
	/* Update any of the values, used when we're incrementing various
	 * fields.
	 */
	mod_cur_headers(pkt_dev);
2883
	queue_map = pkt_dev->cur_queue_map;
2884

S
Stephen Hemminger 已提交
2885 2886 2887
	skb = __netdev_alloc_skb(odev,
				 pkt_dev->cur_pkt_size + 64
				 + 16 + pkt_dev->pkt_overhead, GFP_NOWAIT);
L
Linus Torvalds 已提交
2888 2889 2890 2891
	if (!skb) {
		sprintf(pkt_dev->result, "No memory");
		return NULL;
	}
J
Junchang Wang 已提交
2892
	prefetchw(skb->data);
L
Linus Torvalds 已提交
2893 2894 2895 2896 2897

	skb_reserve(skb, 16);

	/*  Reserve for ethernet and IP header  */
	eth = (__u8 *) skb_push(skb, 14);
2898 2899 2900
	mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32));
	if (pkt_dev->nr_labels)
		mpls_push(mpls, pkt_dev);
F
Francesco Fondelli 已提交
2901 2902

	if (pkt_dev->vlan_id != 0xffff) {
2903
		if (pkt_dev->svlan_id != 0xffff) {
F
Francesco Fondelli 已提交
2904
			svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16));
A
Al Viro 已提交
2905 2906 2907
			*svlan_tci = build_tci(pkt_dev->svlan_id,
					       pkt_dev->svlan_cfi,
					       pkt_dev->svlan_p);
F
Francesco Fondelli 已提交
2908
			svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16));
2909
			*svlan_encapsulated_proto = htons(ETH_P_8021Q);
F
Francesco Fondelli 已提交
2910 2911
		}
		vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16));
A
Al Viro 已提交
2912 2913 2914
		*vlan_tci = build_tci(pkt_dev->vlan_id,
				      pkt_dev->vlan_cfi,
				      pkt_dev->vlan_p);
F
Francesco Fondelli 已提交
2915
		vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16));
2916
		*vlan_encapsulated_proto = htons(ETH_P_IPV6);
F
Francesco Fondelli 已提交
2917 2918
	}

2919
	skb->network_header = skb->tail;
2920
	skb->transport_header = skb->network_header + sizeof(struct ipv6hdr);
2921
	skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr));
2922
	skb_set_queue_mapping(skb, queue_map);
J
John Fastabend 已提交
2923
	skb->priority = pkt_dev->skb_priority;
2924 2925
	iph = ipv6_hdr(skb);
	udph = udp_hdr(skb);
L
Linus Torvalds 已提交
2926 2927

	memcpy(eth, pkt_dev->hh, 12);
2928
	*(__be16 *) &eth[12] = protocol;
2929

2930 2931 2932
	/* Eth + IPh + UDPh + mpls */
	datalen = pkt_dev->cur_pkt_size - 14 -
		  sizeof(struct ipv6hdr) - sizeof(struct udphdr) -
2933
		  pkt_dev->pkt_overhead;
L
Linus Torvalds 已提交
2934

L
Luiz Capitulino 已提交
2935
	if (datalen < sizeof(struct pktgen_hdr)) {
L
Linus Torvalds 已提交
2936 2937
		datalen = sizeof(struct pktgen_hdr);
		if (net_ratelimit())
J
Joe Perches 已提交
2938
			pr_info("increased datalen to %d\n", datalen);
L
Linus Torvalds 已提交
2939 2940 2941 2942
	}

	udph->source = htons(pkt_dev->cur_udp_src);
	udph->dest = htons(pkt_dev->cur_udp_dst);
L
Luiz Capitulino 已提交
2943 2944
	udph->len = htons(datalen + sizeof(struct udphdr));
	udph->check = 0;	/* No checksum */
L
Linus Torvalds 已提交
2945

2946
	*(__be32 *) iph = htonl(0x60000000);	/* Version + flow */
L
Linus Torvalds 已提交
2947

F
Francesco Fondelli 已提交
2948 2949
	if (pkt_dev->traffic_class) {
		/* Version + traffic class + flow (0) */
A
Al Viro 已提交
2950
		*(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20));
F
Francesco Fondelli 已提交
2951 2952
	}

L
Linus Torvalds 已提交
2953 2954 2955 2956 2957 2958 2959 2960
	iph->hop_limit = 32;

	iph->payload_len = htons(sizeof(struct udphdr) + datalen);
	iph->nexthdr = IPPROTO_UDP;

	ipv6_addr_copy(&iph->daddr, &pkt_dev->cur_in6_daddr);
	ipv6_addr_copy(&iph->saddr, &pkt_dev->cur_in6_saddr);

2961
	skb->mac_header = (skb->network_header - ETH_HLEN -
2962
			   pkt_dev->pkt_overhead);
2963
	skb->protocol = protocol;
L
Linus Torvalds 已提交
2964 2965 2966
	skb->dev = odev;
	skb->pkt_type = PACKET_HOST;

E
Eric Dumazet 已提交
2967
	pktgen_finalize_skb(pkt_dev, skb, datalen);
L
Luiz Capitulino 已提交
2968

L
Linus Torvalds 已提交
2969 2970 2971
	return skb;
}

S
Stephen Hemminger 已提交
2972 2973
static struct sk_buff *fill_packet(struct net_device *odev,
				   struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
2974
{
L
Luiz Capitulino 已提交
2975
	if (pkt_dev->flags & F_IPV6)
L
Linus Torvalds 已提交
2976 2977 2978 2979 2980
		return fill_packet_ipv6(odev, pkt_dev);
	else
		return fill_packet_ipv4(odev, pkt_dev);
}

L
Luiz Capitulino 已提交
2981
static void pktgen_clear_counters(struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
2982
{
L
Luiz Capitulino 已提交
2983 2984
	pkt_dev->seq_num = 1;
	pkt_dev->idle_acc = 0;
L
Linus Torvalds 已提交
2985
	pkt_dev->sofar = 0;
L
Luiz Capitulino 已提交
2986 2987
	pkt_dev->tx_bytes = 0;
	pkt_dev->errors = 0;
L
Linus Torvalds 已提交
2988 2989 2990 2991 2992 2993
}

/* Set up structure for sending pkts, clear counters */

static void pktgen_run(struct pktgen_thread *t)
{
2994
	struct pktgen_dev *pkt_dev;
L
Linus Torvalds 已提交
2995 2996
	int started = 0;

J
Joe Perches 已提交
2997
	func_enter();
L
Linus Torvalds 已提交
2998 2999

	if_lock(t);
3000
	list_for_each_entry(pkt_dev, &t->if_list, list) {
L
Linus Torvalds 已提交
3001 3002 3003 3004 3005 3006

		/*
		 * setup odev and create initial packet.
		 */
		pktgen_setup_inject(pkt_dev);

L
Luiz Capitulino 已提交
3007
		if (pkt_dev->odev) {
L
Linus Torvalds 已提交
3008
			pktgen_clear_counters(pkt_dev);
L
Luiz Capitulino 已提交
3009
			pkt_dev->running = 1;	/* Cranke yeself! */
L
Linus Torvalds 已提交
3010
			pkt_dev->skb = NULL;
3011 3012 3013
			pkt_dev->started_at =
				pkt_dev->next_tx = ktime_now();

3014
			set_pkt_overhead(pkt_dev);
L
Luiz Capitulino 已提交
3015

L
Linus Torvalds 已提交
3016 3017
			strcpy(pkt_dev->result, "Starting");
			started++;
L
Luiz Capitulino 已提交
3018
		} else
L
Linus Torvalds 已提交
3019 3020 3021
			strcpy(pkt_dev->result, "Error starting");
	}
	if_unlock(t);
L
Luiz Capitulino 已提交
3022 3023
	if (started)
		t->control &= ~(T_STOP);
L
Linus Torvalds 已提交
3024 3025 3026 3027
}

static void pktgen_stop_all_threads_ifs(void)
{
3028
	struct pktgen_thread *t;
L
Linus Torvalds 已提交
3029

J
Joe Perches 已提交
3030
	func_enter();
L
Linus Torvalds 已提交
3031

3032
	mutex_lock(&pktgen_thread_lock);
3033 3034

	list_for_each_entry(t, &pktgen_threads, th_list)
3035
		t->control |= T_STOP;
3036

3037
	mutex_unlock(&pktgen_thread_lock);
L
Linus Torvalds 已提交
3038 3039
}

S
Stephen Hemminger 已提交
3040
static int thread_is_running(const struct pktgen_thread *t)
L
Linus Torvalds 已提交
3041
{
S
Stephen Hemminger 已提交
3042
	const struct pktgen_dev *pkt_dev;
L
Linus Torvalds 已提交
3043

3044
	list_for_each_entry(pkt_dev, &t->if_list, list)
S
Stephen Hemminger 已提交
3045 3046 3047
		if (pkt_dev->running)
			return 1;
	return 0;
L
Linus Torvalds 已提交
3048 3049
}

L
Luiz Capitulino 已提交
3050
static int pktgen_wait_thread_run(struct pktgen_thread *t)
L
Linus Torvalds 已提交
3051
{
L
Luiz Capitulino 已提交
3052
	if_lock(t);
L
Linus Torvalds 已提交
3053

L
Luiz Capitulino 已提交
3054
	while (thread_is_running(t)) {
L
Linus Torvalds 已提交
3055

L
Luiz Capitulino 已提交
3056
		if_unlock(t);
L
Linus Torvalds 已提交
3057

L
Luiz Capitulino 已提交
3058
		msleep_interruptible(100);
L
Linus Torvalds 已提交
3059

L
Luiz Capitulino 已提交
3060 3061 3062 3063 3064 3065 3066 3067
		if (signal_pending(current))
			goto signal;
		if_lock(t);
	}
	if_unlock(t);
	return 1;
signal:
	return 0;
L
Linus Torvalds 已提交
3068 3069 3070 3071
}

static int pktgen_wait_all_threads_run(void)
{
3072
	struct pktgen_thread *t;
L
Linus Torvalds 已提交
3073
	int sig = 1;
L
Luiz Capitulino 已提交
3074

3075
	mutex_lock(&pktgen_thread_lock);
3076 3077

	list_for_each_entry(t, &pktgen_threads, th_list) {
L
Linus Torvalds 已提交
3078
		sig = pktgen_wait_thread_run(t);
L
Luiz Capitulino 已提交
3079 3080
		if (sig == 0)
			break;
L
Linus Torvalds 已提交
3081
	}
3082 3083 3084

	if (sig == 0)
		list_for_each_entry(t, &pktgen_threads, th_list)
L
Linus Torvalds 已提交
3085
			t->control |= (T_STOP);
3086

3087
	mutex_unlock(&pktgen_thread_lock);
L
Linus Torvalds 已提交
3088 3089 3090 3091 3092
	return sig;
}

static void pktgen_run_all_threads(void)
{
3093
	struct pktgen_thread *t;
L
Linus Torvalds 已提交
3094

J
Joe Perches 已提交
3095
	func_enter();
L
Linus Torvalds 已提交
3096

3097
	mutex_lock(&pktgen_thread_lock);
L
Linus Torvalds 已提交
3098

3099
	list_for_each_entry(t, &pktgen_threads, th_list)
L
Linus Torvalds 已提交
3100
		t->control |= (T_RUN);
3101

3102
	mutex_unlock(&pktgen_thread_lock);
L
Linus Torvalds 已提交
3103

3104 3105
	/* Propagate thread->control  */
	schedule_timeout_interruptible(msecs_to_jiffies(125));
L
Luiz Capitulino 已提交
3106

L
Linus Torvalds 已提交
3107 3108 3109
	pktgen_wait_all_threads_run();
}

3110 3111 3112 3113
static void pktgen_reset_all_threads(void)
{
	struct pktgen_thread *t;

J
Joe Perches 已提交
3114
	func_enter();
3115 3116 3117 3118 3119 3120 3121 3122

	mutex_lock(&pktgen_thread_lock);

	list_for_each_entry(t, &pktgen_threads, th_list)
		t->control |= (T_REMDEVALL);

	mutex_unlock(&pktgen_thread_lock);

3123 3124
	/* Propagate thread->control  */
	schedule_timeout_interruptible(msecs_to_jiffies(125));
3125 3126 3127 3128

	pktgen_wait_all_threads_run();
}

L
Linus Torvalds 已提交
3129 3130
static void show_results(struct pktgen_dev *pkt_dev, int nr_frags)
{
3131
	__u64 bps, mbps, pps;
L
Luiz Capitulino 已提交
3132
	char *p = pkt_dev->result;
3133 3134 3135 3136
	ktime_t elapsed = ktime_sub(pkt_dev->stopped_at,
				    pkt_dev->started_at);
	ktime_t idle = ns_to_ktime(pkt_dev->idle_acc);

3137
	p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n",
3138 3139 3140
		     (unsigned long long)ktime_to_us(elapsed),
		     (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)),
		     (unsigned long long)ktime_to_us(idle),
L
Luiz Capitulino 已提交
3141 3142 3143
		     (unsigned long long)pkt_dev->sofar,
		     pkt_dev->cur_pkt_size, nr_frags);

3144 3145
	pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC,
			ktime_to_ns(elapsed));
L
Luiz Capitulino 已提交
3146 3147 3148 3149 3150 3151 3152 3153 3154 3155

	bps = pps * 8 * pkt_dev->cur_pkt_size;

	mbps = bps;
	do_div(mbps, 1000000);
	p += sprintf(p, "  %llupps %lluMb/sec (%llubps) errors: %llu",
		     (unsigned long long)pps,
		     (unsigned long long)mbps,
		     (unsigned long long)bps,
		     (unsigned long long)pkt_dev->errors);
L
Linus Torvalds 已提交
3156 3157 3158
}

/* Set stopped-at timer, remove from running list, do counters & statistics */
L
Luiz Capitulino 已提交
3159
static int pktgen_stop_device(struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
3160
{
L
Luiz Capitulino 已提交
3161
	int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1;
3162

L
Luiz Capitulino 已提交
3163
	if (!pkt_dev->running) {
J
Joe Perches 已提交
3164 3165
		pr_warning("interface: %s is already stopped\n",
			   pkt_dev->odevname);
L
Luiz Capitulino 已提交
3166 3167
		return -EINVAL;
	}
L
Linus Torvalds 已提交
3168

S
Stephen Hemminger 已提交
3169 3170
	kfree_skb(pkt_dev->skb);
	pkt_dev->skb = NULL;
3171
	pkt_dev->stopped_at = ktime_now();
L
Luiz Capitulino 已提交
3172
	pkt_dev->running = 0;
L
Linus Torvalds 已提交
3173

3174
	show_results(pkt_dev, nr_frags);
L
Linus Torvalds 已提交
3175

L
Luiz Capitulino 已提交
3176
	return 0;
L
Linus Torvalds 已提交
3177 3178
}

L
Luiz Capitulino 已提交
3179
static struct pktgen_dev *next_to_run(struct pktgen_thread *t)
L
Linus Torvalds 已提交
3180
{
3181
	struct pktgen_dev *pkt_dev, *best = NULL;
L
Luiz Capitulino 已提交
3182

L
Linus Torvalds 已提交
3183 3184
	if_lock(t);

3185 3186
	list_for_each_entry(pkt_dev, &t->if_list, list) {
		if (!pkt_dev->running)
L
Luiz Capitulino 已提交
3187 3188
			continue;
		if (best == NULL)
3189
			best = pkt_dev;
3190
		else if (ktime_lt(pkt_dev->next_tx, best->next_tx))
3191
			best = pkt_dev;
L
Linus Torvalds 已提交
3192 3193
	}
	if_unlock(t);
L
Luiz Capitulino 已提交
3194
	return best;
L
Linus Torvalds 已提交
3195 3196
}

L
Luiz Capitulino 已提交
3197 3198
static void pktgen_stop(struct pktgen_thread *t)
{
3199
	struct pktgen_dev *pkt_dev;
L
Linus Torvalds 已提交
3200

J
Joe Perches 已提交
3201
	func_enter();
L
Linus Torvalds 已提交
3202

L
Luiz Capitulino 已提交
3203
	if_lock(t);
L
Linus Torvalds 已提交
3204

3205 3206
	list_for_each_entry(pkt_dev, &t->if_list, list) {
		pktgen_stop_device(pkt_dev);
3207
	}
L
Linus Torvalds 已提交
3208

L
Luiz Capitulino 已提交
3209
	if_unlock(t);
L
Linus Torvalds 已提交
3210 3211
}

3212 3213 3214 3215 3216 3217
/*
 * one of our devices needs to be removed - find it
 * and remove it
 */
static void pktgen_rem_one_if(struct pktgen_thread *t)
{
3218 3219
	struct list_head *q, *n;
	struct pktgen_dev *cur;
3220

J
Joe Perches 已提交
3221
	func_enter();
3222 3223 3224

	if_lock(t);

3225 3226
	list_for_each_safe(q, n, &t->if_list) {
		cur = list_entry(q, struct pktgen_dev, list);
3227

L
Luiz Capitulino 已提交
3228 3229
		if (!cur->removal_mark)
			continue;
3230

3231
		kfree_skb(cur->skb);
3232 3233 3234 3235 3236 3237 3238 3239 3240 3241
		cur->skb = NULL;

		pktgen_remove_device(t, cur);

		break;
	}

	if_unlock(t);
}

L
Luiz Capitulino 已提交
3242
static void pktgen_rem_all_ifs(struct pktgen_thread *t)
L
Linus Torvalds 已提交
3243
{
3244 3245
	struct list_head *q, *n;
	struct pktgen_dev *cur;
L
Luiz Capitulino 已提交
3246

J
Joe Perches 已提交
3247 3248
	func_enter();

L
Luiz Capitulino 已提交
3249
	/* Remove all devices, free mem */
3250

L
Luiz Capitulino 已提交
3251
	if_lock(t);
L
Linus Torvalds 已提交
3252

3253 3254
	list_for_each_safe(q, n, &t->if_list) {
		cur = list_entry(q, struct pktgen_dev, list);
3255

3256
		kfree_skb(cur->skb);
3257 3258
		cur->skb = NULL;

L
Linus Torvalds 已提交
3259 3260 3261
		pktgen_remove_device(t, cur);
	}

L
Luiz Capitulino 已提交
3262
	if_unlock(t);
L
Linus Torvalds 已提交
3263 3264
}

L
Luiz Capitulino 已提交
3265
static void pktgen_rem_thread(struct pktgen_thread *t)
L
Linus Torvalds 已提交
3266
{
L
Luiz Capitulino 已提交
3267
	/* Remove from the thread list */
L
Linus Torvalds 已提交
3268

3269
	remove_proc_entry(t->tsk->comm, pg_proc_dir);
L
Linus Torvalds 已提交
3270 3271 3272

}

3273
static void pktgen_resched(struct pktgen_dev *pkt_dev)
3274
{
3275
	ktime_t idle_start = ktime_now();
3276 3277 3278
	schedule();
	pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_now(), idle_start));
}
3279

3280 3281 3282
static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev)
{
	ktime_t idle_start = ktime_now();
3283

3284 3285 3286 3287 3288 3289 3290 3291 3292
	while (atomic_read(&(pkt_dev->skb->users)) != 1) {
		if (signal_pending(current))
			break;

		if (need_resched())
			pktgen_resched(pkt_dev);
		else
			cpu_relax();
	}
3293
	pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_now(), idle_start));
3294 3295
}

S
Stephen Hemminger 已提交
3296
static void pktgen_xmit(struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
3297
{
3298
	struct net_device *odev = pkt_dev->odev;
3299
	netdev_tx_t (*xmit)(struct sk_buff *, struct net_device *)
3300
		= odev->netdev_ops->ndo_start_xmit;
3301 3302
	struct netdev_queue *txq;
	u16 queue_map;
L
Linus Torvalds 已提交
3303 3304
	int ret;

3305 3306 3307 3308
	/* If device is offline, then don't send */
	if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) {
		pktgen_stop_device(pkt_dev);
		return;
3309 3310
	}

3311 3312 3313 3314 3315
	/* This is max DELAY, this has special meaning of
	 * "never transmit"
	 */
	if (unlikely(pkt_dev->delay == ULLONG_MAX)) {
		pkt_dev->next_tx = ktime_add_ns(ktime_now(), ULONG_MAX);
3316
		return;
L
Linus Torvalds 已提交
3317
	}
L
Luiz Capitulino 已提交
3318

3319
	/* If no skb or clone count exhausted then get new one */
3320 3321 3322 3323 3324 3325 3326
	if (!pkt_dev->skb || (pkt_dev->last_ok &&
			      ++pkt_dev->clone_count >= pkt_dev->clone_skb)) {
		/* build a new pkt */
		kfree_skb(pkt_dev->skb);

		pkt_dev->skb = fill_packet(odev, pkt_dev);
		if (pkt_dev->skb == NULL) {
J
Joe Perches 已提交
3327
			pr_err("ERROR: couldn't allocate skb in fill_packet\n");
3328 3329 3330
			schedule();
			pkt_dev->clone_count--;	/* back out increment, OOM */
			return;
L
Linus Torvalds 已提交
3331
		}
3332
		pkt_dev->last_pkt_size = pkt_dev->skb->len;
3333 3334
		pkt_dev->allocated_skbs++;
		pkt_dev->clone_count = 0;	/* reset counter */
L
Linus Torvalds 已提交
3335
	}
3336

3337 3338 3339
	if (pkt_dev->delay && pkt_dev->last_ok)
		spin(pkt_dev, pkt_dev->next_tx);

3340 3341 3342 3343
	queue_map = skb_get_queue_mapping(pkt_dev->skb);
	txq = netdev_get_tx_queue(odev, queue_map);

	__netif_tx_lock_bh(txq);
3344

3345
	if (unlikely(netif_tx_queue_frozen_or_stopped(txq))) {
3346
		ret = NETDEV_TX_BUSY;
3347 3348 3349 3350 3351
		pkt_dev->last_ok = 0;
		goto unlock;
	}
	atomic_inc(&(pkt_dev->skb->users));
	ret = (*xmit)(pkt_dev->skb, odev);
3352 3353 3354 3355 3356 3357 3358

	switch (ret) {
	case NETDEV_TX_OK:
		txq_trans_update(txq);
		pkt_dev->last_ok = 1;
		pkt_dev->sofar++;
		pkt_dev->seq_num++;
3359
		pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3360
		break;
3361 3362 3363 3364 3365 3366
	case NET_XMIT_DROP:
	case NET_XMIT_CN:
	case NET_XMIT_POLICED:
		/* skb has been consumed */
		pkt_dev->errors++;
		break;
3367 3368
	default: /* Drivers are not supposed to return other values! */
		if (net_ratelimit())
3369
			pr_info("%s xmit error: %d\n", pkt_dev->odevname, ret);
3370 3371 3372 3373 3374 3375 3376
		pkt_dev->errors++;
		/* fallthru */
	case NETDEV_TX_LOCKED:
	case NETDEV_TX_BUSY:
		/* Retry it next time */
		atomic_dec(&(pkt_dev->skb->users));
		pkt_dev->last_ok = 0;
L
Luiz Capitulino 已提交
3377
	}
3378
unlock:
3379
	__netif_tx_unlock_bh(txq);
L
Luiz Capitulino 已提交
3380

L
Linus Torvalds 已提交
3381 3382
	/* If pkt_dev->count is zero, then run forever */
	if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) {
3383
		pktgen_wait_for_skb(pkt_dev);
L
Luiz Capitulino 已提交
3384

L
Linus Torvalds 已提交
3385 3386
		/* Done with this */
		pktgen_stop_device(pkt_dev);
L
Luiz Capitulino 已提交
3387 3388
	}
}
L
Linus Torvalds 已提交
3389

3390
/*
L
Linus Torvalds 已提交
3391 3392 3393
 * Main loop of the thread goes here
 */

3394
static int pktgen_thread_worker(void *arg)
L
Linus Torvalds 已提交
3395 3396
{
	DEFINE_WAIT(wait);
3397
	struct pktgen_thread *t = arg;
L
Luiz Capitulino 已提交
3398
	struct pktgen_dev *pkt_dev = NULL;
L
Linus Torvalds 已提交
3399 3400
	int cpu = t->cpu;

3401
	BUG_ON(smp_processor_id() != cpu);
L
Linus Torvalds 已提交
3402 3403

	init_waitqueue_head(&t->queue);
3404
	complete(&t->start_done);
L
Linus Torvalds 已提交
3405

J
Joe Perches 已提交
3406
	pr_debug("starting pktgen/%d:  pid=%d\n", cpu, task_pid_nr(current));
L
Linus Torvalds 已提交
3407

3408
	set_current_state(TASK_INTERRUPTIBLE);
L
Luiz Capitulino 已提交
3409

3410 3411
	set_freezable();

3412 3413
	while (!kthread_should_stop()) {
		pkt_dev = next_to_run(t);
L
Linus Torvalds 已提交
3414

3415
		if (unlikely(!pkt_dev && t->control == 0)) {
E
Eric Dumazet 已提交
3416 3417
			if (pktgen_exiting)
				break;
3418 3419 3420
			wait_event_interruptible_timeout(t->queue,
							 t->control != 0,
							 HZ/10);
R
Rafael J. Wysocki 已提交
3421
			try_to_freeze();
3422
			continue;
3423
		}
L
Linus Torvalds 已提交
3424

3425
		__set_current_state(TASK_RUNNING);
L
Luiz Capitulino 已提交
3426

3427
		if (likely(pkt_dev)) {
L
Linus Torvalds 已提交
3428 3429
			pktgen_xmit(pkt_dev);

3430 3431 3432 3433 3434 3435
			if (need_resched())
				pktgen_resched(pkt_dev);
			else
				cpu_relax();
		}

L
Luiz Capitulino 已提交
3436
		if (t->control & T_STOP) {
L
Linus Torvalds 已提交
3437 3438 3439 3440
			pktgen_stop(t);
			t->control &= ~(T_STOP);
		}

L
Luiz Capitulino 已提交
3441
		if (t->control & T_RUN) {
L
Linus Torvalds 已提交
3442 3443 3444 3445
			pktgen_run(t);
			t->control &= ~(T_RUN);
		}

L
Luiz Capitulino 已提交
3446
		if (t->control & T_REMDEVALL) {
L
Linus Torvalds 已提交
3447
			pktgen_rem_all_ifs(t);
3448 3449 3450
			t->control &= ~(T_REMDEVALL);
		}

L
Luiz Capitulino 已提交
3451
		if (t->control & T_REMDEV) {
3452
			pktgen_rem_one_if(t);
L
Linus Torvalds 已提交
3453 3454 3455
			t->control &= ~(T_REMDEV);
		}

A
Andrew Morton 已提交
3456 3457
		try_to_freeze();

3458
		set_current_state(TASK_INTERRUPTIBLE);
L
Luiz Capitulino 已提交
3459
	}
L
Linus Torvalds 已提交
3460

J
Joe Perches 已提交
3461
	pr_debug("%s stopping all device\n", t->tsk->comm);
L
Luiz Capitulino 已提交
3462
	pktgen_stop(t);
L
Linus Torvalds 已提交
3463

J
Joe Perches 已提交
3464
	pr_debug("%s removing all device\n", t->tsk->comm);
L
Luiz Capitulino 已提交
3465
	pktgen_rem_all_ifs(t);
L
Linus Torvalds 已提交
3466

J
Joe Perches 已提交
3467
	pr_debug("%s removing thread\n", t->tsk->comm);
L
Luiz Capitulino 已提交
3468
	pktgen_rem_thread(t);
3469

E
Eric Dumazet 已提交
3470 3471 3472 3473 3474 3475 3476
	/* Wait for kthread_stop */
	while (!kthread_should_stop()) {
		set_current_state(TASK_INTERRUPTIBLE);
		schedule();
	}
	__set_current_state(TASK_RUNNING);

3477
	return 0;
L
Linus Torvalds 已提交
3478 3479
}

L
Luiz Capitulino 已提交
3480
static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
E
Eric Dumazet 已提交
3481
					  const char *ifname, bool exact)
L
Linus Torvalds 已提交
3482
{
3483
	struct pktgen_dev *p, *pkt_dev = NULL;
E
Eric Dumazet 已提交
3484
	size_t len = strlen(ifname);
L
Luiz Capitulino 已提交
3485

E
Eric Dumazet 已提交
3486
	if_lock(t);
3487
	list_for_each_entry(p, &t->if_list, list)
E
Eric Dumazet 已提交
3488 3489 3490 3491 3492
		if (strncmp(p->odevname, ifname, len) == 0) {
			if (p->odevname[len]) {
				if (exact || p->odevname[len] != '@')
					continue;
			}
3493
			pkt_dev = p;
L
Luiz Capitulino 已提交
3494 3495 3496 3497
			break;
		}

	if_unlock(t);
J
Joe Perches 已提交
3498
	pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev);
L
Luiz Capitulino 已提交
3499
	return pkt_dev;
L
Linus Torvalds 已提交
3500 3501
}

3502 3503
/*
 * Adds a dev at front of if_list.
L
Linus Torvalds 已提交
3504 3505
 */

L
Luiz Capitulino 已提交
3506 3507
static int add_dev_to_thread(struct pktgen_thread *t,
			     struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
3508 3509
{
	int rv = 0;
L
Luiz Capitulino 已提交
3510 3511 3512 3513

	if_lock(t);

	if (pkt_dev->pg_thread) {
J
Joe Perches 已提交
3514
		pr_err("ERROR: already assigned to a thread\n");
L
Luiz Capitulino 已提交
3515 3516 3517
		rv = -EBUSY;
		goto out;
	}
3518 3519

	list_add(&pkt_dev->list, &t->if_list);
L
Luiz Capitulino 已提交
3520
	pkt_dev->pg_thread = t;
L
Linus Torvalds 已提交
3521 3522
	pkt_dev->running = 0;

L
Luiz Capitulino 已提交
3523 3524 3525
out:
	if_unlock(t);
	return rv;
L
Linus Torvalds 已提交
3526 3527 3528 3529
}

/* Called under thread lock */

L
Luiz Capitulino 已提交
3530
static int pktgen_add_device(struct pktgen_thread *t, const char *ifname)
L
Linus Torvalds 已提交
3531
{
L
Luiz Capitulino 已提交
3532
	struct pktgen_dev *pkt_dev;
3533
	int err;
E
Eric Dumazet 已提交
3534
	int node = cpu_to_node(t->cpu);
L
Luiz Capitulino 已提交
3535

L
Linus Torvalds 已提交
3536 3537
	/* We don't allow a device to be on several threads */

3538 3539
	pkt_dev = __pktgen_NN_threads(ifname, FIND);
	if (pkt_dev) {
J
Joe Perches 已提交
3540
		pr_err("ERROR: interface already used\n");
L
Luiz Capitulino 已提交
3541 3542
		return -EBUSY;
	}
3543

E
Eric Dumazet 已提交
3544
	pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node);
3545 3546 3547
	if (!pkt_dev)
		return -ENOMEM;

E
Eric Dumazet 已提交
3548
	strcpy(pkt_dev->odevname, ifname);
3549
	pkt_dev->flows = vzalloc_node(MAX_CFLOWS * sizeof(struct flow_state),
E
Eric Dumazet 已提交
3550
				      node);
3551 3552 3553 3554 3555
	if (pkt_dev->flows == NULL) {
		kfree(pkt_dev);
		return -ENOMEM;
	}

3556
	pkt_dev->removal_mark = 0;
3557 3558 3559
	pkt_dev->min_pkt_size = ETH_ZLEN;
	pkt_dev->max_pkt_size = ETH_ZLEN;
	pkt_dev->nfrags = 0;
3560
	pkt_dev->delay = pg_delay_d;
3561 3562
	pkt_dev->count = pg_count_d;
	pkt_dev->sofar = 0;
L
Luiz Capitulino 已提交
3563
	pkt_dev->udp_src_min = 9;	/* sink port */
3564 3565 3566
	pkt_dev->udp_src_max = 9;
	pkt_dev->udp_dst_min = 9;
	pkt_dev->udp_dst_max = 9;
F
Francesco Fondelli 已提交
3567 3568 3569 3570 3571 3572
	pkt_dev->vlan_p = 0;
	pkt_dev->vlan_cfi = 0;
	pkt_dev->vlan_id = 0xffff;
	pkt_dev->svlan_p = 0;
	pkt_dev->svlan_cfi = 0;
	pkt_dev->svlan_id = 0xffff;
R
Robert Olsson 已提交
3573
	pkt_dev->node = -1;
F
Francesco Fondelli 已提交
3574

3575 3576 3577
	err = pktgen_setup_dev(pkt_dev, ifname);
	if (err)
		goto out1;
3578 3579
	if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)
		pkt_dev->clone_skb = pg_clone_skb_d;
3580

3581 3582
	pkt_dev->entry = proc_create_data(ifname, 0600, pg_proc_dir,
					  &pktgen_if_fops, pkt_dev);
3583
	if (!pkt_dev->entry) {
J
Joe Perches 已提交
3584
		pr_err("cannot create %s/%s procfs entry\n",
3585
		       PG_PROC_DIR, ifname);
3586 3587
		err = -EINVAL;
		goto out2;
3588
	}
J
Jamal Hadi Salim 已提交
3589 3590 3591 3592
#ifdef CONFIG_XFRM
	pkt_dev->ipsmode = XFRM_MODE_TRANSPORT;
	pkt_dev->ipsproto = IPPROTO_ESP;
#endif
3593 3594

	return add_dev_to_thread(t, pkt_dev);
3595 3596 3597
out2:
	dev_put(pkt_dev->odev);
out1:
J
Jamal Hadi Salim 已提交
3598 3599 3600
#ifdef CONFIG_XFRM
	free_SAs(pkt_dev);
#endif
F
Figo.zhang 已提交
3601
	vfree(pkt_dev->flows);
3602 3603
	kfree(pkt_dev);
	return err;
L
Linus Torvalds 已提交
3604 3605
}

3606
static int __init pktgen_create_thread(int cpu)
L
Linus Torvalds 已提交
3607
{
3608
	struct pktgen_thread *t;
3609
	struct proc_dir_entry *pe;
3610
	struct task_struct *p;
L
Luiz Capitulino 已提交
3611

E
Eric Dumazet 已提交
3612 3613
	t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL,
			 cpu_to_node(cpu));
L
Luiz Capitulino 已提交
3614
	if (!t) {
J
Joe Perches 已提交
3615
		pr_err("ERROR: out of memory, can't create new thread\n");
L
Luiz Capitulino 已提交
3616 3617 3618 3619
		return -ENOMEM;
	}

	spin_lock_init(&t->if_lock);
L
Linus Torvalds 已提交
3620
	t->cpu = cpu;
L
Luiz Capitulino 已提交
3621

3622 3623 3624
	INIT_LIST_HEAD(&t->if_list);

	list_add_tail(&t->th_list, &pktgen_threads);
3625
	init_completion(&t->start_done);
3626

3627 3628 3629 3630
	p = kthread_create_on_node(pktgen_thread_worker,
				   t,
				   cpu_to_node(cpu),
				   "kpktgend_%d", cpu);
3631
	if (IS_ERR(p)) {
J
Joe Perches 已提交
3632
		pr_err("kernel_thread() failed for cpu %d\n", t->cpu);
3633 3634 3635 3636 3637 3638 3639
		list_del(&t->th_list);
		kfree(t);
		return PTR_ERR(p);
	}
	kthread_bind(p, cpu);
	t->tsk = p;

3640 3641
	pe = proc_create_data(t->tsk->comm, 0600, pg_proc_dir,
			      &pktgen_thread_fops, t);
L
Luiz Capitulino 已提交
3642
	if (!pe) {
J
Joe Perches 已提交
3643
		pr_err("cannot create %s/%s procfs entry\n",
3644 3645 3646
		       PG_PROC_DIR, t->tsk->comm);
		kthread_stop(p);
		list_del(&t->th_list);
L
Luiz Capitulino 已提交
3647 3648 3649
		kfree(t);
		return -EINVAL;
	}
3650

3651
	wake_up_process(p);
3652
	wait_for_completion(&t->start_done);
L
Linus Torvalds 已提交
3653 3654 3655 3656

	return 0;
}

3657 3658
/*
 * Removes a device from the thread if_list.
L
Linus Torvalds 已提交
3659
 */
L
Luiz Capitulino 已提交
3660 3661
static void _rem_dev_from_if_list(struct pktgen_thread *t,
				  struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
3662
{
3663 3664
	struct list_head *q, *n;
	struct pktgen_dev *p;
L
Linus Torvalds 已提交
3665

3666 3667 3668 3669
	list_for_each_safe(q, n, &t->if_list) {
		p = list_entry(q, struct pktgen_dev, list);
		if (p == pkt_dev)
			list_del(&p->list);
L
Linus Torvalds 已提交
3670 3671 3672
	}
}

L
Luiz Capitulino 已提交
3673 3674
static int pktgen_remove_device(struct pktgen_thread *t,
				struct pktgen_dev *pkt_dev)
L
Linus Torvalds 已提交
3675 3676
{

J
Joe Perches 已提交
3677
	pr_debug("remove_device pkt_dev=%p\n", pkt_dev);
L
Linus Torvalds 已提交
3678

L
Luiz Capitulino 已提交
3679
	if (pkt_dev->running) {
J
Joe Perches 已提交
3680
		pr_warning("WARNING: trying to remove a running interface, stopping it now\n");
L
Luiz Capitulino 已提交
3681 3682 3683 3684
		pktgen_stop_device(pkt_dev);
	}

	/* Dis-associate from the interface */
L
Linus Torvalds 已提交
3685 3686 3687

	if (pkt_dev->odev) {
		dev_put(pkt_dev->odev);
L
Luiz Capitulino 已提交
3688 3689 3690
		pkt_dev->odev = NULL;
	}

L
Linus Torvalds 已提交
3691 3692 3693 3694
	/* And update the thread if_list */

	_rem_dev_from_if_list(t, pkt_dev);

3695 3696
	if (pkt_dev->entry)
		remove_proc_entry(pkt_dev->entry->name, pg_proc_dir);
L
Linus Torvalds 已提交
3697

J
Jamal Hadi Salim 已提交
3698 3699 3700
#ifdef CONFIG_XFRM
	free_SAs(pkt_dev);
#endif
F
Figo.zhang 已提交
3701
	vfree(pkt_dev->flows);
E
Eric Dumazet 已提交
3702 3703
	if (pkt_dev->page)
		put_page(pkt_dev->page);
L
Linus Torvalds 已提交
3704
	kfree(pkt_dev);
L
Luiz Capitulino 已提交
3705
	return 0;
L
Linus Torvalds 已提交
3706 3707
}

L
Luiz Capitulino 已提交
3708
static int __init pg_init(void)
L
Linus Torvalds 已提交
3709 3710
{
	int cpu;
3711
	struct proc_dir_entry *pe;
W
WANG Cong 已提交
3712
	int ret = 0;
L
Linus Torvalds 已提交
3713

J
Joe Perches 已提交
3714
	pr_info("%s", version);
L
Linus Torvalds 已提交
3715

3716
	pg_proc_dir = proc_mkdir(PG_PROC_DIR, init_net.proc_net);
3717 3718
	if (!pg_proc_dir)
		return -ENODEV;
L
Linus Torvalds 已提交
3719

3720
	pe = proc_create(PGCTRL, 0600, pg_proc_dir, &pktgen_fops);
L
Luiz Capitulino 已提交
3721
	if (pe == NULL) {
J
Joe Perches 已提交
3722
		pr_err("ERROR: cannot create %s procfs entry\n", PGCTRL);
W
WANG Cong 已提交
3723 3724
		ret = -EINVAL;
		goto remove_dir;
L
Luiz Capitulino 已提交
3725
	}
L
Linus Torvalds 已提交
3726 3727

	register_netdevice_notifier(&pktgen_notifier_block);
L
Luiz Capitulino 已提交
3728

J
John Hawkes 已提交
3729
	for_each_online_cpu(cpu) {
3730
		int err;
L
Linus Torvalds 已提交
3731

3732
		err = pktgen_create_thread(cpu);
3733
		if (err)
J
Joe Perches 已提交
3734 3735
			pr_warning("WARNING: Cannot create thread for cpu %d (%d)\n",
				   cpu, err);
L
Luiz Capitulino 已提交
3736
	}
3737 3738

	if (list_empty(&pktgen_threads)) {
J
Joe Perches 已提交
3739
		pr_err("ERROR: Initialization failed for all threads\n");
W
WANG Cong 已提交
3740 3741
		ret = -ENODEV;
		goto unregister;
3742 3743
	}

L
Luiz Capitulino 已提交
3744
	return 0;
W
WANG Cong 已提交
3745 3746 3747 3748 3749 3750 3751

 unregister:
	unregister_netdevice_notifier(&pktgen_notifier_block);
	remove_proc_entry(PGCTRL, pg_proc_dir);
 remove_dir:
	proc_net_remove(&init_net, PG_PROC_DIR);
	return ret;
L
Linus Torvalds 已提交
3752 3753 3754 3755
}

static void __exit pg_cleanup(void)
{
3756 3757
	struct pktgen_thread *t;
	struct list_head *q, *n;
L
Linus Torvalds 已提交
3758

L
Luiz Capitulino 已提交
3759
	/* Stop all interfaces & threads */
E
Eric Dumazet 已提交
3760
	pktgen_exiting = true;
L
Linus Torvalds 已提交
3761

3762 3763
	list_for_each_safe(q, n, &pktgen_threads) {
		t = list_entry(q, struct pktgen_thread, th_list);
3764 3765
		kthread_stop(t->tsk);
		kfree(t);
L
Luiz Capitulino 已提交
3766
	}
L
Linus Torvalds 已提交
3767

L
Luiz Capitulino 已提交
3768
	/* Un-register us from receiving netdevice events */
L
Linus Torvalds 已提交
3769 3770
	unregister_netdevice_notifier(&pktgen_notifier_block);

L
Luiz Capitulino 已提交
3771
	/* Clean up proc file system */
3772
	remove_proc_entry(PGCTRL, pg_proc_dir);
3773
	proc_net_remove(&init_net, PG_PROC_DIR);
L
Linus Torvalds 已提交
3774 3775 3776 3777 3778
}

module_init(pg_init);
module_exit(pg_cleanup);

S
Stephen Hemminger 已提交
3779
MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>");
L
Linus Torvalds 已提交
3780 3781
MODULE_DESCRIPTION("Packet Generator tool");
MODULE_LICENSE("GPL");
S
Stephen Hemminger 已提交
3782
MODULE_VERSION(VERSION);
L
Linus Torvalds 已提交
3783
module_param(pg_count_d, int, 0);
S
Stephen Hemminger 已提交
3784
MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject");
L
Linus Torvalds 已提交
3785
module_param(pg_delay_d, int, 0);
S
Stephen Hemminger 已提交
3786
MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)");
L
Linus Torvalds 已提交
3787
module_param(pg_clone_skb_d, int, 0);
S
Stephen Hemminger 已提交
3788
MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet");
L
Linus Torvalds 已提交
3789
module_param(debug, int, 0);
S
Stephen Hemminger 已提交
3790
MODULE_PARM_DESC(debug, "Enable debugging of pktgen module");