Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/arch/x86/net/bpf_jit_comp.c

  1 /* bpf_jit_comp.c : BPF JIT compiler
  2  *
  3  * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
  4  *
  5  * This program is free software; you can redistribute it and/or
  6  * modify it under the terms of the GNU General Public License
  7  * as published by the Free Software Foundation; version 2
  8  * of the License.
  9  */
 10 #include <linux/moduleloader.h>
 11 #include <asm/cacheflush.h>
 12 #include <linux/netdevice.h>
 13 #include <linux/filter.h>
 14 #include <linux/if_vlan.h>
 15 #include <linux/random.h>
 16 
 17 /*
 18  * Conventions :
 19  *  EAX : BPF A accumulator
 20  *  EBX : BPF X accumulator
 21  *  RDI : pointer to skb   (first argument given to JIT function)
 22  *  RBP : frame pointer (even if CONFIG_FRAME_POINTER=n)
 23  *  ECX,EDX,ESI : scratch registers
 24  *  r9d : skb->len - skb->data_len (headlen)
 25  *  r8  : skb->data
 26  * -8(RBP) : saved RBX value
 27  * -16(RBP)..-80(RBP) : BPF_MEMWORDS values
 28  */
 29 int bpf_jit_enable __read_mostly;
 30 
 31 /*
 32  * assembly code in arch/x86/net/bpf_jit.S
 33  */
 34 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
 35 extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[];
 36 extern u8 sk_load_byte_positive_offset[], sk_load_byte_msh_positive_offset[];
 37 extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[];
 38 extern u8 sk_load_byte_negative_offset[], sk_load_byte_msh_negative_offset[];
 39 
 40 static inline u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
 41 {
 42         if (len == 1)
 43                 *ptr = bytes;
 44         else if (len == 2)
 45                 *(u16 *)ptr = bytes;
 46         else {
 47                 *(u32 *)ptr = bytes;
 48                 barrier();
 49         }
 50         return ptr + len;
 51 }
 52 
 53 #define EMIT(bytes, len)        do { prog = emit_code(prog, bytes, len); } while (0)
 54 
 55 #define EMIT1(b1)               EMIT(b1, 1)
 56 #define EMIT2(b1, b2)           EMIT((b1) + ((b2) << 8), 2)
 57 #define EMIT3(b1, b2, b3)       EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
 58 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
 59 #define EMIT1_off32(b1, off)    do { EMIT1(b1); EMIT(off, 4);} while (0)
 60 
 61 #define CLEAR_A() EMIT2(0x31, 0xc0) /* xor %eax,%eax */
 62 #define CLEAR_X() EMIT2(0x31, 0xdb) /* xor %ebx,%ebx */
 63 
 64 static inline bool is_imm8(int value)
 65 {
 66         return value <= 127 && value >= -128;
 67 }
 68 
 69 static inline bool is_near(int offset)
 70 {
 71         return offset <= 127 && offset >= -128;
 72 }
 73 
 74 #define EMIT_JMP(offset)                                                \
 75 do {                                                                    \
 76         if (offset) {                                                   \
 77                 if (is_near(offset))                                    \
 78                         EMIT2(0xeb, offset); /* jmp .+off8 */           \
 79                 else                                                    \
 80                         EMIT1_off32(0xe9, offset); /* jmp .+off32 */    \
 81         }                                                               \
 82 } while (0)
 83 
 84 /* list of x86 cond jumps opcodes (. + s8)
 85  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
 86  */
 87 #define X86_JB  0x72
 88 #define X86_JAE 0x73
 89 #define X86_JE  0x74
 90 #define X86_JNE 0x75
 91 #define X86_JBE 0x76
 92 #define X86_JA  0x77
 93 
 94 #define EMIT_COND_JMP(op, offset)                               \
 95 do {                                                            \
 96         if (is_near(offset))                                    \
 97                 EMIT2(op, offset); /* jxx .+off8 */             \
 98         else {                                                  \
 99                 EMIT2(0x0f, op + 0x10);                         \
100                 EMIT(offset, 4); /* jxx .+off32 */              \
101         }                                                       \
102 } while (0)
103 
104 #define COND_SEL(CODE, TOP, FOP)        \
105         case CODE:                      \
106                 t_op = TOP;             \
107                 f_op = FOP;             \
108                 goto cond_branch
109 
110 
111 #define SEEN_DATAREF 1 /* might call external helpers */
112 #define SEEN_XREG    2 /* ebx is used */
113 #define SEEN_MEM     4 /* use mem[] for temporary storage */
114 
115 static inline void bpf_flush_icache(void *start, void *end)
116 {
117         mm_segment_t old_fs = get_fs();
118 
119         set_fs(KERNEL_DS);
120         smp_wmb();
121         flush_icache_range((unsigned long)start, (unsigned long)end);
122         set_fs(old_fs);
123 }
124 
125 #define CHOOSE_LOAD_FUNC(K, func) \
126         ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
127 
128 /* Helper to find the offset of pkt_type in sk_buff
129  * We want to make sure its still a 3bit field starting at a byte boundary.
130  */
131 #define PKT_TYPE_MAX 7
132 static int pkt_type_offset(void)
133 {
134         struct sk_buff skb_probe = {
135                 .pkt_type = ~0,
136         };
137         char *ct = (char *)&skb_probe;
138         unsigned int off;
139 
140         for (off = 0; off < sizeof(struct sk_buff); off++) {
141                 if (ct[off] == PKT_TYPE_MAX)
142                         return off;
143         }
144         pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n");
145         return -1;
146 }
147 
148 struct bpf_binary_header {
149         unsigned int    pages;
150         /* Note : for security reasons, bpf code will follow a randomly
151          * sized amount of int3 instructions
152          */
153         u8              image[];
154 };
155 
156 static struct bpf_binary_header *bpf_alloc_binary(unsigned int proglen,
157                                                   u8 **image_ptr)
158 {
159         unsigned int sz, hole;
160         struct bpf_binary_header *header;
161 
162         /* Most of BPF filters are really small,
163          * but if some of them fill a page, allow at least
164          * 128 extra bytes to insert a random section of int3
165          */
166         sz = round_up(proglen + sizeof(*header) + 128, PAGE_SIZE);
167         header = module_alloc(sz);
168         if (!header)
169                 return NULL;
170 
171         memset(header, 0xcc, sz); /* fill whole space with int3 instructions */
172 
173         header->pages = sz / PAGE_SIZE;
174         hole = sz - (proglen + sizeof(*header));
175 
176         /* insert a random number of int3 instructions before BPF code */
177         *image_ptr = &header->image[prandom_u32() % hole];
178         return header;
179 }
180 
181 void bpf_jit_compile(struct sk_filter *fp)
182 {
183         u8 temp[64];
184         u8 *prog;
185         unsigned int proglen, oldproglen = 0;
186         int ilen, i;
187         int t_offset, f_offset;
188         u8 t_op, f_op, seen = 0, pass;
189         u8 *image = NULL;
190         struct bpf_binary_header *header = NULL;
191         u8 *func;
192         int pc_ret0 = -1; /* bpf index of first RET #0 instruction (if any) */
193         unsigned int cleanup_addr; /* epilogue code offset */
194         unsigned int *addrs;
195         const struct sock_filter *filter = fp->insns;
196         int flen = fp->len;
197 
198         if (!bpf_jit_enable)
199                 return;
200 
201         addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL);
202         if (addrs == NULL)
203                 return;
204 
205         /* Before first pass, make a rough estimation of addrs[]
206          * each bpf instruction is translated to less than 64 bytes
207          */
208         for (proglen = 0, i = 0; i < flen; i++) {
209                 proglen += 64;
210                 addrs[i] = proglen;
211         }
212         cleanup_addr = proglen; /* epilogue address */
213 
214         for (pass = 0; pass < 10; pass++) {
215                 u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen;
216                 /* no prologue/epilogue for trivial filters (RET something) */
217                 proglen = 0;
218                 prog = temp;
219 
220                 if (seen_or_pass0) {
221                         EMIT4(0x55, 0x48, 0x89, 0xe5); /* push %rbp; mov %rsp,%rbp */
222                         EMIT4(0x48, 0x83, 0xec, 96);    /* subq  $96,%rsp       */
223                         /* note : must save %rbx in case bpf_error is hit */
224                         if (seen_or_pass0 & (SEEN_XREG | SEEN_DATAREF))
225                                 EMIT4(0x48, 0x89, 0x5d, 0xf8); /* mov %rbx, -8(%rbp) */
226                         if (seen_or_pass0 & SEEN_XREG)
227                                 CLEAR_X(); /* make sure we dont leek kernel memory */
228 
229                         /*
230                          * If this filter needs to access skb data,
231                          * loads r9 and r8 with :
232                          *  r9 = skb->len - skb->data_len
233                          *  r8 = skb->data
234                          */
235                         if (seen_or_pass0 & SEEN_DATAREF) {
236                                 if (offsetof(struct sk_buff, len) <= 127)
237                                         /* mov    off8(%rdi),%r9d */
238                                         EMIT4(0x44, 0x8b, 0x4f, offsetof(struct sk_buff, len));
239                                 else {
240                                         /* mov    off32(%rdi),%r9d */
241                                         EMIT3(0x44, 0x8b, 0x8f);
242                                         EMIT(offsetof(struct sk_buff, len), 4);
243                                 }
244                                 if (is_imm8(offsetof(struct sk_buff, data_len)))
245                                         /* sub    off8(%rdi),%r9d */
246                                         EMIT4(0x44, 0x2b, 0x4f, offsetof(struct sk_buff, data_len));
247                                 else {
248                                         EMIT3(0x44, 0x2b, 0x8f);
249                                         EMIT(offsetof(struct sk_buff, data_len), 4);
250                                 }
251 
252                                 if (is_imm8(offsetof(struct sk_buff, data)))
253                                         /* mov off8(%rdi),%r8 */
254                                         EMIT4(0x4c, 0x8b, 0x47, offsetof(struct sk_buff, data));
255                                 else {
256                                         /* mov off32(%rdi),%r8 */
257                                         EMIT3(0x4c, 0x8b, 0x87);
258                                         EMIT(offsetof(struct sk_buff, data), 4);
259                                 }
260                         }
261                 }
262 
263                 switch (filter[0].code) {
264                 case BPF_S_RET_K:
265                 case BPF_S_LD_W_LEN:
266                 case BPF_S_ANC_PROTOCOL:
267                 case BPF_S_ANC_IFINDEX:
268                 case BPF_S_ANC_MARK:
269                 case BPF_S_ANC_RXHASH:
270                 case BPF_S_ANC_CPU:
271                 case BPF_S_ANC_VLAN_TAG:
272                 case BPF_S_ANC_VLAN_TAG_PRESENT:
273                 case BPF_S_ANC_QUEUE:
274                 case BPF_S_ANC_PKTTYPE:
275                 case BPF_S_LD_W_ABS:
276                 case BPF_S_LD_H_ABS:
277                 case BPF_S_LD_B_ABS:
278                         /* first instruction sets A register (or is RET 'constant') */
279                         break;
280                 default:
281                         /* make sure we dont leak kernel information to user */
282                         CLEAR_A(); /* A = 0 */
283                 }
284 
285                 for (i = 0; i < flen; i++) {
286                         unsigned int K = filter[i].k;
287 
288                         switch (filter[i].code) {
289                         case BPF_S_ALU_ADD_X: /* A += X; */
290                                 seen |= SEEN_XREG;
291                                 EMIT2(0x01, 0xd8);              /* add %ebx,%eax */
292                                 break;
293                         case BPF_S_ALU_ADD_K: /* A += K; */
294                                 if (!K)
295                                         break;
296                                 if (is_imm8(K))
297                                         EMIT3(0x83, 0xc0, K);   /* add imm8,%eax */
298                                 else
299                                         EMIT1_off32(0x05, K);   /* add imm32,%eax */
300                                 break;
301                         case BPF_S_ALU_SUB_X: /* A -= X; */
302                                 seen |= SEEN_XREG;
303                                 EMIT2(0x29, 0xd8);              /* sub    %ebx,%eax */
304                                 break;
305                         case BPF_S_ALU_SUB_K: /* A -= K */
306                                 if (!K)
307                                         break;
308                                 if (is_imm8(K))
309                                         EMIT3(0x83, 0xe8, K); /* sub imm8,%eax */
310                                 else
311                                         EMIT1_off32(0x2d, K); /* sub imm32,%eax */
312                                 break;
313                         case BPF_S_ALU_MUL_X: /* A *= X; */
314                                 seen |= SEEN_XREG;
315                                 EMIT3(0x0f, 0xaf, 0xc3);        /* imul %ebx,%eax */
316                                 break;
317                         case BPF_S_ALU_MUL_K: /* A *= K */
318                                 if (is_imm8(K))
319                                         EMIT3(0x6b, 0xc0, K); /* imul imm8,%eax,%eax */
320                                 else {
321                                         EMIT2(0x69, 0xc0);              /* imul imm32,%eax */
322                                         EMIT(K, 4);
323                                 }
324                                 break;
325                         case BPF_S_ALU_DIV_X: /* A /= X; */
326                                 seen |= SEEN_XREG;
327                                 EMIT2(0x85, 0xdb);      /* test %ebx,%ebx */
328                                 if (pc_ret0 > 0) {
329                                         /* addrs[pc_ret0 - 1] is start address of target
330                                          * (addrs[i] - 4) is the address following this jmp
331                                          * ("xor %edx,%edx; div %ebx" being 4 bytes long)
332                                          */
333                                         EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
334                                                                 (addrs[i] - 4));
335                                 } else {
336                                         EMIT_COND_JMP(X86_JNE, 2 + 5);
337                                         CLEAR_A();
338                                         EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 4)); /* jmp .+off32 */
339                                 }
340                                 EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */
341                                 break;
342                         case BPF_S_ALU_MOD_X: /* A %= X; */
343                                 seen |= SEEN_XREG;
344                                 EMIT2(0x85, 0xdb);      /* test %ebx,%ebx */
345                                 if (pc_ret0 > 0) {
346                                         /* addrs[pc_ret0 - 1] is start address of target
347                                          * (addrs[i] - 6) is the address following this jmp
348                                          * ("xor %edx,%edx; div %ebx;mov %edx,%eax" being 6 bytes long)
349                                          */
350                                         EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
351                                                                 (addrs[i] - 6));
352                                 } else {
353                                         EMIT_COND_JMP(X86_JNE, 2 + 5);
354                                         CLEAR_A();
355                                         EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 6)); /* jmp .+off32 */
356                                 }
357                                 EMIT2(0x31, 0xd2);      /* xor %edx,%edx */
358                                 EMIT2(0xf7, 0xf3);      /* div %ebx */
359                                 EMIT2(0x89, 0xd0);      /* mov %edx,%eax */
360                                 break;
361                         case BPF_S_ALU_MOD_K: /* A %= K; */
362                                 if (K == 1) {
363                                         CLEAR_A();
364                                         break;
365                                 }
366                                 EMIT2(0x31, 0xd2);      /* xor %edx,%edx */
367                                 EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
368                                 EMIT2(0xf7, 0xf1);      /* div %ecx */
369                                 EMIT2(0x89, 0xd0);      /* mov %edx,%eax */
370                                 break;
371                         case BPF_S_ALU_DIV_K: /* A /= K */
372                                 if (K == 1)
373                                         break;
374                                 EMIT2(0x31, 0xd2);      /* xor %edx,%edx */
375                                 EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
376                                 EMIT2(0xf7, 0xf1);      /* div %ecx */
377                                 break;
378                         case BPF_S_ALU_AND_X:
379                                 seen |= SEEN_XREG;
380                                 EMIT2(0x21, 0xd8);              /* and %ebx,%eax */
381                                 break;
382                         case BPF_S_ALU_AND_K:
383                                 if (K >= 0xFFFFFF00) {
384                                         EMIT2(0x24, K & 0xFF); /* and imm8,%al */
385                                 } else if (K >= 0xFFFF0000) {
386                                         EMIT2(0x66, 0x25);      /* and imm16,%ax */
387                                         EMIT(K, 2);
388                                 } else {
389                                         EMIT1_off32(0x25, K);   /* and imm32,%eax */
390                                 }
391                                 break;
392                         case BPF_S_ALU_OR_X:
393                                 seen |= SEEN_XREG;
394                                 EMIT2(0x09, 0xd8);              /* or %ebx,%eax */
395                                 break;
396                         case BPF_S_ALU_OR_K:
397                                 if (is_imm8(K))
398                                         EMIT3(0x83, 0xc8, K); /* or imm8,%eax */
399                                 else
400                                         EMIT1_off32(0x0d, K);   /* or imm32,%eax */
401                                 break;
402                         case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */
403                         case BPF_S_ALU_XOR_X:
404                                 seen |= SEEN_XREG;
405                                 EMIT2(0x31, 0xd8);              /* xor %ebx,%eax */
406                                 break;
407                         case BPF_S_ALU_XOR_K: /* A ^= K; */
408                                 if (K == 0)
409                                         break;
410                                 if (is_imm8(K))
411                                         EMIT3(0x83, 0xf0, K);   /* xor imm8,%eax */
412                                 else
413                                         EMIT1_off32(0x35, K);   /* xor imm32,%eax */
414                                 break;
415                         case BPF_S_ALU_LSH_X: /* A <<= X; */
416                                 seen |= SEEN_XREG;
417                                 EMIT4(0x89, 0xd9, 0xd3, 0xe0);  /* mov %ebx,%ecx; shl %cl,%eax */
418                                 break;
419                         case BPF_S_ALU_LSH_K:
420                                 if (K == 0)
421                                         break;
422                                 else if (K == 1)
423                                         EMIT2(0xd1, 0xe0); /* shl %eax */
424                                 else
425                                         EMIT3(0xc1, 0xe0, K);
426                                 break;
427                         case BPF_S_ALU_RSH_X: /* A >>= X; */
428                                 seen |= SEEN_XREG;
429                                 EMIT4(0x89, 0xd9, 0xd3, 0xe8);  /* mov %ebx,%ecx; shr %cl,%eax */
430                                 break;
431                         case BPF_S_ALU_RSH_K: /* A >>= K; */
432                                 if (K == 0)
433                                         break;
434                                 else if (K == 1)
435                                         EMIT2(0xd1, 0xe8); /* shr %eax */
436                                 else
437                                         EMIT3(0xc1, 0xe8, K);
438                                 break;
439                         case BPF_S_ALU_NEG:
440                                 EMIT2(0xf7, 0xd8);              /* neg %eax */
441                                 break;
442                         case BPF_S_RET_K:
443                                 if (!K) {
444                                         if (pc_ret0 == -1)
445                                                 pc_ret0 = i;
446                                         CLEAR_A();
447                                 } else {
448                                         EMIT1_off32(0xb8, K);   /* mov $imm32,%eax */
449                                 }
450                                 /* fallinto */
451                         case BPF_S_RET_A:
452                                 if (seen_or_pass0) {
453                                         if (i != flen - 1) {
454                                                 EMIT_JMP(cleanup_addr - addrs[i]);
455                                                 break;
456                                         }
457                                         if (seen_or_pass0 & SEEN_XREG)
458                                                 EMIT4(0x48, 0x8b, 0x5d, 0xf8);  /* mov  -8(%rbp),%rbx */
459                                         EMIT1(0xc9);            /* leaveq */
460                                 }
461                                 EMIT1(0xc3);            /* ret */
462                                 break;
463                         case BPF_S_MISC_TAX: /* X = A */
464                                 seen |= SEEN_XREG;
465                                 EMIT2(0x89, 0xc3);      /* mov    %eax,%ebx */
466                                 break;
467                         case BPF_S_MISC_TXA: /* A = X */
468                                 seen |= SEEN_XREG;
469                                 EMIT2(0x89, 0xd8);      /* mov    %ebx,%eax */
470                                 break;
471                         case BPF_S_LD_IMM: /* A = K */
472                                 if (!K)
473                                         CLEAR_A();
474                                 else
475                                         EMIT1_off32(0xb8, K); /* mov $imm32,%eax */
476                                 break;
477                         case BPF_S_LDX_IMM: /* X = K */
478                                 seen |= SEEN_XREG;
479                                 if (!K)
480                                         CLEAR_X();
481                                 else
482                                         EMIT1_off32(0xbb, K); /* mov $imm32,%ebx */
483                                 break;
484                         case BPF_S_LD_MEM: /* A = mem[K] : mov off8(%rbp),%eax */
485                                 seen |= SEEN_MEM;
486                                 EMIT3(0x8b, 0x45, 0xf0 - K*4);
487                                 break;
488                         case BPF_S_LDX_MEM: /* X = mem[K] : mov off8(%rbp),%ebx */
489                                 seen |= SEEN_XREG | SEEN_MEM;
490                                 EMIT3(0x8b, 0x5d, 0xf0 - K*4);
491                                 break;
492                         case BPF_S_ST: /* mem[K] = A : mov %eax,off8(%rbp) */
493                                 seen |= SEEN_MEM;
494                                 EMIT3(0x89, 0x45, 0xf0 - K*4);
495                                 break;
496                         case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
497                                 seen |= SEEN_XREG | SEEN_MEM;
498                                 EMIT3(0x89, 0x5d, 0xf0 - K*4);
499                                 break;
500                         case BPF_S_LD_W_LEN: /* A = skb->len; */
501                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
502                                 if (is_imm8(offsetof(struct sk_buff, len)))
503                                         /* mov    off8(%rdi),%eax */
504                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, len));
505                                 else {
506                                         EMIT2(0x8b, 0x87);
507                                         EMIT(offsetof(struct sk_buff, len), 4);
508                                 }
509                                 break;
510                         case BPF_S_LDX_W_LEN: /* X = skb->len; */
511                                 seen |= SEEN_XREG;
512                                 if (is_imm8(offsetof(struct sk_buff, len)))
513                                         /* mov off8(%rdi),%ebx */
514                                         EMIT3(0x8b, 0x5f, offsetof(struct sk_buff, len));
515                                 else {
516                                         EMIT2(0x8b, 0x9f);
517                                         EMIT(offsetof(struct sk_buff, len), 4);
518                                 }
519                                 break;
520                         case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */
521                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
522                                 if (is_imm8(offsetof(struct sk_buff, protocol))) {
523                                         /* movzwl off8(%rdi),%eax */
524                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, protocol));
525                                 } else {
526                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
527                                         EMIT(offsetof(struct sk_buff, protocol), 4);
528                                 }
529                                 EMIT2(0x86, 0xc4); /* ntohs() : xchg   %al,%ah */
530                                 break;
531                         case BPF_S_ANC_IFINDEX:
532                                 if (is_imm8(offsetof(struct sk_buff, dev))) {
533                                         /* movq off8(%rdi),%rax */
534                                         EMIT4(0x48, 0x8b, 0x47, offsetof(struct sk_buff, dev));
535                                 } else {
536                                         EMIT3(0x48, 0x8b, 0x87); /* movq off32(%rdi),%rax */
537                                         EMIT(offsetof(struct sk_buff, dev), 4);
538                                 }
539                                 EMIT3(0x48, 0x85, 0xc0);        /* test %rax,%rax */
540                                 EMIT_COND_JMP(X86_JE, cleanup_addr - (addrs[i] - 6));
541                                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
542                                 EMIT2(0x8b, 0x80);      /* mov off32(%rax),%eax */
543                                 EMIT(offsetof(struct net_device, ifindex), 4);
544                                 break;
545                         case BPF_S_ANC_MARK:
546                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
547                                 if (is_imm8(offsetof(struct sk_buff, mark))) {
548                                         /* mov off8(%rdi),%eax */
549                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, mark));
550                                 } else {
551                                         EMIT2(0x8b, 0x87);
552                                         EMIT(offsetof(struct sk_buff, mark), 4);
553                                 }
554                                 break;
555                         case BPF_S_ANC_RXHASH:
556                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
557                                 if (is_imm8(offsetof(struct sk_buff, rxhash))) {
558                                         /* mov off8(%rdi),%eax */
559                                         EMIT3(0x8b, 0x47, offsetof(struct sk_buff, rxhash));
560                                 } else {
561                                         EMIT2(0x8b, 0x87);
562                                         EMIT(offsetof(struct sk_buff, rxhash), 4);
563                                 }
564                                 break;
565                         case BPF_S_ANC_QUEUE:
566                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
567                                 if (is_imm8(offsetof(struct sk_buff, queue_mapping))) {
568                                         /* movzwl off8(%rdi),%eax */
569                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, queue_mapping));
570                                 } else {
571                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
572                                         EMIT(offsetof(struct sk_buff, queue_mapping), 4);
573                                 }
574                                 break;
575                         case BPF_S_ANC_CPU:
576 #ifdef CONFIG_SMP
577                                 EMIT4(0x65, 0x8b, 0x04, 0x25); /* mov %gs:off32,%eax */
578                                 EMIT((u32)(unsigned long)&cpu_number, 4); /* A = smp_processor_id(); */
579 #else
580                                 CLEAR_A();
581 #endif
582                                 break;
583                         case BPF_S_ANC_VLAN_TAG:
584                         case BPF_S_ANC_VLAN_TAG_PRESENT:
585                                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
586                                 if (is_imm8(offsetof(struct sk_buff, vlan_tci))) {
587                                         /* movzwl off8(%rdi),%eax */
588                                         EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, vlan_tci));
589                                 } else {
590                                         EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
591                                         EMIT(offsetof(struct sk_buff, vlan_tci), 4);
592                                 }
593                                 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
594                                 if (filter[i].code == BPF_S_ANC_VLAN_TAG) {
595                                         EMIT3(0x80, 0xe4, 0xef); /* and    $0xef,%ah */
596                                 } else {
597                                         EMIT3(0xc1, 0xe8, 0x0c); /* shr    $0xc,%eax */
598                                         EMIT3(0x83, 0xe0, 0x01); /* and    $0x1,%eax */
599                                 }
600                                 break;
601                         case BPF_S_ANC_PKTTYPE:
602                         {
603                                 int off = pkt_type_offset();
604 
605                                 if (off < 0)
606                                         goto out;
607                                 if (is_imm8(off)) {
608                                         /* movzbl off8(%rdi),%eax */
609                                         EMIT4(0x0f, 0xb6, 0x47, off);
610                                 } else {
611                                         /* movbl off32(%rdi),%eax */
612                                         EMIT3(0x0f, 0xb6, 0x87);
613                                         EMIT(off, 4);
614                                 }
615                                 EMIT3(0x83, 0xe0, PKT_TYPE_MAX); /* and    $0x7,%eax */
616                                 break;
617                         }
618                         case BPF_S_LD_W_ABS:
619                                 func = CHOOSE_LOAD_FUNC(K, sk_load_word);
620 common_load:                    seen |= SEEN_DATAREF;
621                                 t_offset = func - (image + addrs[i]);
622                                 EMIT1_off32(0xbe, K); /* mov imm32,%esi */
623                                 EMIT1_off32(0xe8, t_offset); /* call */
624                                 break;
625                         case BPF_S_LD_H_ABS:
626                                 func = CHOOSE_LOAD_FUNC(K, sk_load_half);
627                                 goto common_load;
628                         case BPF_S_LD_B_ABS:
629                                 func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
630                                 goto common_load;
631                         case BPF_S_LDX_B_MSH:
632                                 func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
633                                 seen |= SEEN_DATAREF | SEEN_XREG;
634                                 t_offset = func - (image + addrs[i]);
635                                 EMIT1_off32(0xbe, K);   /* mov imm32,%esi */
636                                 EMIT1_off32(0xe8, t_offset); /* call sk_load_byte_msh */
637                                 break;
638                         case BPF_S_LD_W_IND:
639                                 func = sk_load_word;
640 common_load_ind:                seen |= SEEN_DATAREF | SEEN_XREG;
641                                 t_offset = func - (image + addrs[i]);
642                                 if (K) {
643                                         if (is_imm8(K)) {
644                                                 EMIT3(0x8d, 0x73, K); /* lea imm8(%rbx), %esi */
645                                         } else {
646                                                 EMIT2(0x8d, 0xb3); /* lea imm32(%rbx),%esi */
647                                                 EMIT(K, 4);
648                                         }
649                                 } else {
650                                         EMIT2(0x89,0xde); /* mov %ebx,%esi */
651                                 }
652                                 EMIT1_off32(0xe8, t_offset);    /* call sk_load_xxx_ind */
653                                 break;
654                         case BPF_S_LD_H_IND:
655                                 func = sk_load_half;
656                                 goto common_load_ind;
657                         case BPF_S_LD_B_IND:
658                                 func = sk_load_byte;
659                                 goto common_load_ind;
660                         case BPF_S_JMP_JA:
661                                 t_offset = addrs[i + K] - addrs[i];
662                                 EMIT_JMP(t_offset);
663                                 break;
664                         COND_SEL(BPF_S_JMP_JGT_K, X86_JA, X86_JBE);
665                         COND_SEL(BPF_S_JMP_JGE_K, X86_JAE, X86_JB);
666                         COND_SEL(BPF_S_JMP_JEQ_K, X86_JE, X86_JNE);
667                         COND_SEL(BPF_S_JMP_JSET_K,X86_JNE, X86_JE);
668                         COND_SEL(BPF_S_JMP_JGT_X, X86_JA, X86_JBE);
669                         COND_SEL(BPF_S_JMP_JGE_X, X86_JAE, X86_JB);
670                         COND_SEL(BPF_S_JMP_JEQ_X, X86_JE, X86_JNE);
671                         COND_SEL(BPF_S_JMP_JSET_X,X86_JNE, X86_JE);
672 
673 cond_branch:                    f_offset = addrs[i + filter[i].jf] - addrs[i];
674                                 t_offset = addrs[i + filter[i].jt] - addrs[i];
675 
676                                 /* same targets, can avoid doing the test :) */
677                                 if (filter[i].jt == filter[i].jf) {
678                                         EMIT_JMP(t_offset);
679                                         break;
680                                 }
681 
682                                 switch (filter[i].code) {
683                                 case BPF_S_JMP_JGT_X:
684                                 case BPF_S_JMP_JGE_X:
685                                 case BPF_S_JMP_JEQ_X:
686                                         seen |= SEEN_XREG;
687                                         EMIT2(0x39, 0xd8); /* cmp %ebx,%eax */
688                                         break;
689                                 case BPF_S_JMP_JSET_X:
690                                         seen |= SEEN_XREG;
691                                         EMIT2(0x85, 0xd8); /* test %ebx,%eax */
692                                         break;
693                                 case BPF_S_JMP_JEQ_K:
694                                         if (K == 0) {
695                                                 EMIT2(0x85, 0xc0); /* test   %eax,%eax */
696                                                 break;
697                                         }
698                                 case BPF_S_JMP_JGT_K:
699                                 case BPF_S_JMP_JGE_K:
700                                         if (K <= 127)
701                                                 EMIT3(0x83, 0xf8, K); /* cmp imm8,%eax */
702                                         else
703                                                 EMIT1_off32(0x3d, K); /* cmp imm32,%eax */
704                                         break;
705                                 case BPF_S_JMP_JSET_K:
706                                         if (K <= 0xFF)
707                                                 EMIT2(0xa8, K); /* test imm8,%al */
708                                         else if (!(K & 0xFFFF00FF))
709                                                 EMIT3(0xf6, 0xc4, K >> 8); /* test imm8,%ah */
710                                         else if (K <= 0xFFFF) {
711                                                 EMIT2(0x66, 0xa9); /* test imm16,%ax */
712                                                 EMIT(K, 2);
713                                         } else {
714                                                 EMIT1_off32(0xa9, K); /* test imm32,%eax */
715                                         }
716                                         break;
717                                 }
718                                 if (filter[i].jt != 0) {
719                                         if (filter[i].jf && f_offset)
720                                                 t_offset += is_near(f_offset) ? 2 : 5;
721                                         EMIT_COND_JMP(t_op, t_offset);
722                                         if (filter[i].jf)
723                                                 EMIT_JMP(f_offset);
724                                         break;
725                                 }
726                                 EMIT_COND_JMP(f_op, f_offset);
727                                 break;
728                         default:
729                                 /* hmm, too complex filter, give up with jit compiler */
730                                 goto out;
731                         }
732                         ilen = prog - temp;
733                         if (image) {
734                                 if (unlikely(proglen + ilen > oldproglen)) {
735                                         pr_err("bpb_jit_compile fatal error\n");
736                                         kfree(addrs);
737                                         module_free(NULL, header);
738                                         return;
739                                 }
740                                 memcpy(image + proglen, temp, ilen);
741                         }
742                         proglen += ilen;
743                         addrs[i] = proglen;
744                         prog = temp;
745                 }
746                 /* last bpf instruction is always a RET :
747                  * use it to give the cleanup instruction(s) addr
748                  */
749                 cleanup_addr = proglen - 1; /* ret */
750                 if (seen_or_pass0)
751                         cleanup_addr -= 1; /* leaveq */
752                 if (seen_or_pass0 & SEEN_XREG)
753                         cleanup_addr -= 4; /* mov  -8(%rbp),%rbx */
754 
755                 if (image) {
756                         if (proglen != oldproglen)
757                                 pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen);
758                         break;
759                 }
760                 if (proglen == oldproglen) {
761                         header = bpf_alloc_binary(proglen, &image);
762                         if (!header)
763                                 goto out;
764                 }
765                 oldproglen = proglen;
766         }
767 
768         if (bpf_jit_enable > 1)
769                 bpf_jit_dump(flen, proglen, pass, image);
770 
771         if (image) {
772                 bpf_flush_icache(header, image + proglen);
773                 set_memory_ro((unsigned long)header, header->pages);
774                 fp->bpf_func = (void *)image;
775         }
776 out:
777         kfree(addrs);
778         return;
779 }
780 
781 static void bpf_jit_free_deferred(struct work_struct *work)
782 {
783         struct sk_filter *fp = container_of(work, struct sk_filter, work);
784         unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
785         struct bpf_binary_header *header = (void *)addr;
786 
787         set_memory_rw(addr, header->pages);
788         module_free(NULL, header);
789         kfree(fp);
790 }
791 
792 void bpf_jit_free(struct sk_filter *fp)
793 {
794         if (fp->bpf_func != sk_run_filter) {
795                 INIT_WORK(&fp->work, bpf_jit_free_deferred);
796                 schedule_work(&fp->work);
797         } else {
798                 kfree(fp);
799         }
800 }
801 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us