linux_dma.c (31426B)
1 /* 2 * Copyright 2017 Broadcom 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License, version 2, as 6 * published by the Free Software Foundation (the "GPL"). 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License version 2 (GPLv2) for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * version 2 (GPLv2) along with this source code. 15 */ 16 /* 17 * $Id: linux_dma.c,v 1.414 Broadcom SDK $ 18 * $Copyright: (c) 2016 Broadcom Corp. 19 * All Rights Reserved.$ 20 * 21 * Linux Kernel BDE DMA memory allocation 22 * 23 * 24 * DMA memory allocation modes 25 * =========================== 26 * 27 * 1. Using private pool in kernel memory 28 * -------------------------------------- 29 * In this mode the BDE module will try to assemble a physically contiguous 30 * of memory using the kernel page allocator. This memory block is then 31 * administered by the mpool allocation functions. Note that once a system 32 * has been running for a while, the memory fragmentation may prevent the 33 * allocator from assembling a contiguous memory block, however, if the 34 * module is loaded shortly after system startup, it is very unlikely to 35 * fail. 36 * 37 * This allocation method is used by default. 38 * 39 * 2. Using private pool in high memory 40 * ------------------------------------ 41 * In this mode the BDE module will assume that unused physical memory is 42 * present at the high_memory address, i.e. memory not managed by the Linux 43 * memory manager. This memory block is mapped into kernel space and 44 * administered by the mpool allocation functions. High memory must be 45 * reserved using either the mem=xxx kernel parameter (recommended), or by 46 * hardcoding the memory limit in the kernel image. 47 * 48 * The module parameter himem=1 enables this allocation mode. 49 * 50 * 3. Using kernel allocators (kmalloc, __get_free_pages) 51 * ------------------------------------------------------ 52 * In this mode all DMA memory is allocated from the kernel on the fly, i.e. 53 * no private DMA memory pool will be created. If large memory blocks are 54 * only allocated at system startup (or not at all), this allocation method 55 * is the most flexible and memory-efficient, however, it is not recommended 56 * for non-coherent memory platforms due to an overall system performance 57 * degradation arising from the use of cache flush/invalidate instructions. 58 * 59 * The module parameter dmasize=0M enables this allocation mode, however if 60 * DMA memory is requested from a user mode application, a private memory 61 * pool will be created and used irrespectively. 62 */ 63 64 #include <gmodule.h> 65 #include <linux-bde.h> 66 #include <linux_dma.h> 67 #include <mpool.h> 68 #include <sdk_config.h> 69 70 #if defined(IPROC_CMICD) && defined(CONFIG_OF) 71 #include <linux/of.h> 72 #endif 73 74 #ifdef BCM_PLX9656_LOCAL_BUS 75 #include <asm/cacheflush.h> 76 #endif 77 78 /* allocation types/methods for the DMA memory pool */ 79 #define ALLOC_TYPE_CHUNK 0 /* use small allocations and join them */ 80 #define ALLOC_TYPE_API 1 /* use one allocation */ 81 82 #if _SIMPLE_MEMORY_ALLOCATION_ 83 #include <linux/dma-mapping.h> 84 #if defined(CONFIG_CMA) && defined(CONFIG_CMA_SIZE_MBYTES) 85 #define DMA_MAX_ALLOC_SIZE (CONFIG_CMA_SIZE_MBYTES * 1024 * 1024) 86 #else 87 #define DMA_MAX_ALLOC_SIZE (1 << (MAX_ORDER - 1 + PAGE_SHIFT)) /* Maximum size the kernel can allocate in one allocation */ 88 #endif 89 #endif /* _SIMPLE_MEMORY_ALLOCATION_ */ 90 91 #if _SIMPLE_MEMORY_ALLOCATION_ == 1 92 /* Use Linux DMA API to allocate contiguous memory */ 93 #define ALLOC_METHOD_DEFAULT ALLOC_TYPE_API 94 #if defined(__arm__) 95 #define USE_DMA_MMAP_COHERENT 96 #define _PGPROT_NONCACHED(x) x = pgprot_noncached((x)) 97 #elif defined(__aarch64__ ) 98 #define USE_DMA_MMAP_COHERENT 99 #define _PGPROT_NONCACHED(x) x = pgprot_writecombine((x)) 100 #endif 101 #else 102 #define ALLOC_METHOD_DEFAULT ALLOC_TYPE_CHUNK 103 #endif 104 105 #ifndef _PGPROT_NONCACHED 106 #ifdef REMAP_DMA_NONCACHED 107 #define _PGPROT_NONCACHED(x) x = pgprot_noncached((x)) 108 #else 109 #define _PGPROT_NONCACHED(x) 110 #endif 111 #endif 112 113 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) 114 #include <linux/slab.h> 115 #define virt_to_bus virt_to_phys 116 #define bus_to_virt phys_to_virt 117 #endif 118 119 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21)) 120 #define VIRT_TO_PAGE(p) virt_to_page((void*)(p)) 121 #else 122 #define VIRT_TO_PAGE(p) virt_to_page((p)) 123 #endif 124 125 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27)) 126 #define BDE_DMA_MAPPING_ERROR(d, p) dma_mapping_error((d),(p)) 127 #else 128 #define BDE_DMA_MAPPING_ERROR(d, p) dma_mapping_error((p)) 129 #endif 130 131 #ifndef KMALLOC_MAX_SIZE 132 #define KMALLOC_MAX_SIZE (1UL << (MAX_ORDER - 1 + PAGE_SHIFT)) 133 #endif 134 135 /* Compatibility */ 136 #ifdef LKM_2_4 137 #define MEM_MAP_RESERVE mem_map_reserve 138 #define MEM_MAP_UNRESERVE mem_map_unreserve 139 #else /* LKM_2_6 */ 140 #define MEM_MAP_RESERVE SetPageReserved 141 #define MEM_MAP_UNRESERVE ClearPageReserved 142 #endif /* LKM_2_x */ 143 144 #ifndef GFP_DMA32 145 #define GFP_DMA32 0 146 #endif 147 148 /* Flags for memory allocations */ 149 #ifdef SAL_BDE_XLP 150 static int mem_flags = GFP_ATOMIC | GFP_KERNEL | GFP_DMA; 151 #else 152 #if defined(CONFIG_ZONE_DMA32) 153 static int mem_flags = GFP_ATOMIC | GFP_DMA32; 154 #else 155 static int mem_flags = GFP_ATOMIC | GFP_DMA; 156 #endif 157 #endif 158 159 /* Debug output */ 160 static int dma_debug = 0; 161 module_param(dma_debug, int, 0); 162 MODULE_PARM_DESC(dma_debug, 163 "DMA debug output enable (default 0)."); 164 165 /* DMA memory pool size */ 166 static char *dmasize; 167 LKM_MOD_PARAM(dmasize, "s", charp, 0); 168 MODULE_PARM_DESC(dmasize, 169 "Specify DMA memory size (default 4MB)"); 170 171 /* Select DMA memory pool allocation method */ 172 static int dmaalloc = ALLOC_METHOD_DEFAULT; 173 LKM_MOD_PARAM(dmaalloc, "i", int, 0); 174 MODULE_PARM_DESC(dmaalloc, "Select DMA memory allocation method"); 175 176 /* Use high memory for DMA */ 177 static char *himem; 178 LKM_MOD_PARAM(himem, "s", charp, 0); 179 MODULE_PARM_DESC(himem, 180 "Use high memory for DMA (default no)"); 181 182 /* Physical high memory address to use for DMA */ 183 static char *himemaddr = 0; 184 LKM_MOD_PARAM(himemaddr, "s", charp, 0); 185 MODULE_PARM_DESC(himemaddr, 186 "Physical address to use for high memory DMA"); 187 188 /* DMA memory allocation */ 189 190 #define ONE_KB 1024 191 #define ONE_MB (1024*1024) 192 #define ONE_GB (1024*1024*1024) 193 194 /* Default DMA memory size */ 195 #ifdef SAL_BDE_DMA_MEM_DEFAULT 196 #define DMA_MEM_DEFAULT (SAL_BDE_DMA_MEM_DEFAULT * ONE_MB) 197 #else 198 #define DMA_MEM_DEFAULT (8 * ONE_MB) 199 #endif 200 201 /* We try to assemble a contiguous segment from chunks of this size */ 202 #define DMA_BLOCK_SIZE (512 * ONE_KB) 203 204 typedef struct _dma_segment { 205 struct list_head list; 206 unsigned long req_size; /* Requested DMA segment size */ 207 unsigned long blk_size; /* DMA block size */ 208 unsigned long blk_order; /* DMA block size in alternate format */ 209 unsigned long seg_size; /* Current DMA segment size */ 210 unsigned long seg_begin; /* Logical address of segment */ 211 unsigned long seg_end; /* Logical end address of segment */ 212 unsigned long *blk_ptr; /* Array of logical DMA block addresses */ 213 int blk_cnt_max; /* Maximum number of block to allocate */ 214 int blk_cnt; /* Current number of blocks allocated */ 215 } dma_segment_t; 216 217 static unsigned int _dma_mem_size = DMA_MEM_DEFAULT; 218 static mpool_handle_t _dma_pool = NULL; 219 static void __iomem *_dma_vbase = NULL; 220 /* cpu physical address for mmap */ 221 static phys_addr_t _cpu_pbase = 0; 222 /* 223 * DMA bus address, it is either identical to cpu physical address 224 * or another address(IOVA) translated by IOMMU. 225 */ 226 static phys_addr_t _dma_pbase = 0; 227 static int _use_himem = 0; 228 static unsigned long _himemaddr = 0; 229 static int _use_dma_mapping = 0; 230 static LIST_HEAD(_dma_seg); 231 232 #define DMA_DEV_INDEX 0 /* Device index to allocate memory pool */ 233 #define DMA_DEV(n) lkbde_get_dma_dev(n) 234 #define BDE_NUM_DEVICES(t) lkbde_get_num_devices(t) 235 236 /* 237 * Function: _find_largest_segment 238 * 239 * Purpose: 240 * Find largest contiguous segment from a pool of DMA blocks. 241 * Parameters: 242 * dseg - DMA segment descriptor 243 * Returns: 244 * 0 on success, < 0 on error. 245 * Notes: 246 * Assembly stops if a segment of the requested segment size 247 * has been obtained. 248 * 249 * Lower address bits of the DMA blocks are used as follows: 250 * 0: Untagged 251 * 1: Discarded block 252 * 2: Part of largest contiguous segment 253 * 3: Part of current contiguous segment 254 */ 255 static int 256 _find_largest_segment(dma_segment_t *dseg) 257 { 258 int i, j, blks, found; 259 unsigned long b, e, a; 260 261 blks = dseg->blk_cnt; 262 /* Clear all block tags */ 263 for (i = 0; i < blks; i++) { 264 dseg->blk_ptr[i] &= ~3; 265 } 266 for (i = 0; i < blks && dseg->seg_size < dseg->req_size; i++) { 267 /* First block must be an untagged block */ 268 if ((dseg->blk_ptr[i] & 3) == 0) { 269 /* Initial segment size is the block size */ 270 b = dseg->blk_ptr[i]; 271 e = b + dseg->blk_size; 272 dseg->blk_ptr[i] |= 3; 273 /* Loop looking for adjacent blocks */ 274 do { 275 found = 0; 276 for (j = i + 1; j < blks && (e - b) < dseg->req_size; j++) { 277 a = dseg->blk_ptr[j]; 278 /* Check untagged blocks only */ 279 if ((a & 3) == 0) { 280 if (a == (b - dseg->blk_size)) { 281 /* Found adjacent block below current segment */ 282 dseg->blk_ptr[j] |= 3; 283 b = a; 284 found = 1; 285 } else if (a == e) { 286 /* Found adjacent block above current segment */ 287 dseg->blk_ptr[j] |= 3; 288 e += dseg->blk_size; 289 found = 1; 290 } 291 } 292 } 293 } while (found); 294 if ((e - b) > dseg->seg_size) { 295 /* The current block is largest so far */ 296 dseg->seg_begin = b; 297 dseg->seg_end = e; 298 dseg->seg_size = e - b; 299 /* Re-tag current and previous largest segment */ 300 for (j = 0; j < blks; j++) { 301 if ((dseg->blk_ptr[j] & 3) == 3) { 302 /* Tag current segment as the largest */ 303 dseg->blk_ptr[j] &= ~1; 304 } else if ((dseg->blk_ptr[j] & 3) == 2) { 305 /* Discard previous largest segment */ 306 dseg->blk_ptr[j] ^= 3; 307 } 308 } 309 } else { 310 /* Discard all blocks in current segment */ 311 for (j = 0; j < blks; j++) { 312 if ((dseg->blk_ptr[j] & 3) == 3) { 313 dseg->blk_ptr[j] &= ~2; 314 } 315 } 316 } 317 } 318 } 319 return 0; 320 } 321 322 /* 323 * Function: _alloc_dma_blocks 324 * 325 * Purpose: 326 * Allocate DMA blocks and add them to the pool. 327 * Parameters: 328 * dseg - DMA segment descriptor 329 * blks - number of DMA blocks to allocate 330 * Returns: 331 * 0 on success, < 0 on error. 332 * Notes: 333 * DMA blocks are allocated using the page allocator. 334 */ 335 static int 336 _alloc_dma_blocks(dma_segment_t *dseg, int blks) 337 { 338 int i, start; 339 unsigned long addr; 340 341 if (dseg->blk_cnt + blks > dseg->blk_cnt_max) { 342 gprintk("No more DMA blocks\n"); 343 return -1; 344 } 345 start = dseg->blk_cnt; 346 for (i = 0; i < blks; i++) { 347 /* 348 * Note that we cannot use pci_alloc_consistent when we 349 * want to be able to map DMA memory to user space. 350 * 351 * The GFP_DMA flag is omitted as this imposes the ISA 352 * addressing limitations on x86 platforms. As long as 353 * we have less than 1GB of memory, we can do PCI DMA 354 * to all physical RAM locations. 355 */ 356 addr = __get_free_pages(mem_flags, dseg->blk_order); 357 if (addr) { 358 dseg->blk_ptr[start + i] = addr; 359 ++dseg->blk_cnt; 360 } else { 361 gprintk("DMA allocation failed: allocated %d of %d " 362 "requested blocks\n", i, blks); 363 return -1; 364 } 365 } 366 return 0; 367 } 368 369 /* 370 * Function: _dma_segment_alloc 371 * 372 * Purpose: 373 * Allocate large physically contiguous DMA segment. 374 * Parameters: 375 * size - requested DMA segment size 376 * blk_size - assemble segment from blocks of this size 377 * Returns: 378 * DMA segment descriptor. 379 * Notes: 380 * Since we cannot allocate large blocks of contiguous 381 * memory from the kernel, we simply keep allocating 382 * smaller chunks until we can assemble a contiguous 383 * block of the desired size. 384 * 385 * When system allowed maximum bytes of memory has been allocated 386 * without a successful assembly of a contiguous DMA 387 * segment, the allocation function will return the 388 * largest contiguous segment found so far. It is up 389 * to the calling function to decide whether this 390 * amount is sufficient to proceed. 391 */ 392 static dma_segment_t * 393 _dma_segment_alloc(size_t size, size_t blk_size) 394 { 395 dma_segment_t *dseg; 396 int i, blk_ptr_size; 397 unsigned long page_addr; 398 struct sysinfo si; 399 400 /* Sanity check */ 401 if (size == 0 || blk_size == 0) { 402 return NULL; 403 } 404 /* Allocate an initialize DMA segment descriptor */ 405 if ((dseg = kmalloc(sizeof(dma_segment_t), GFP_KERNEL)) == NULL) { 406 return NULL; 407 } 408 memset(dseg, 0, sizeof(dma_segment_t)); 409 dseg->req_size = size; 410 dseg->blk_size = PAGE_ALIGN(blk_size); 411 while ((PAGE_SIZE << dseg->blk_order) < dseg->blk_size) { 412 dseg->blk_order++; 413 } 414 415 si_meminfo(&si); 416 dseg->blk_cnt_max = (si.totalram << PAGE_SHIFT) / dseg->blk_size; 417 blk_ptr_size = dseg->blk_cnt_max * sizeof(unsigned long); 418 if (blk_ptr_size > KMALLOC_MAX_SIZE) { 419 blk_ptr_size = KMALLOC_MAX_SIZE; 420 dseg->blk_cnt_max = KMALLOC_MAX_SIZE / sizeof(unsigned long); 421 } 422 /* Allocate an initialize DMA block pool */ 423 dseg->blk_ptr = KMALLOC(blk_ptr_size, GFP_KERNEL); 424 if (dseg->blk_ptr == NULL) { 425 kfree(dseg); 426 return NULL; 427 } 428 memset(dseg->blk_ptr, 0, blk_ptr_size); 429 /* Allocate minimum number of blocks */ 430 if (_alloc_dma_blocks(dseg, dseg->req_size / dseg->blk_size) != 0) { 431 gprintk("Failed to allocate minimum number of DMA blocks\n"); 432 /* 433 * _alloc_dma_blocks() returns -1 if it fails to allocate the requested 434 * number of blocks, but it may still have allocated something. Fall 435 * through and return dseg filled in with as much memory as we could 436 * allocate. 437 */ 438 } 439 /* Allocate more blocks until we have a complete segment */ 440 do { 441 _find_largest_segment(dseg); 442 if (dseg->seg_size >= dseg->req_size) { 443 break; 444 } 445 } while (_alloc_dma_blocks(dseg, 8) == 0); 446 /* Reserve all pages in the DMA segment and free unused blocks */ 447 for (i = 0; i < dseg->blk_cnt; i++) { 448 if ((dseg->blk_ptr[i] & 3) == 2) { 449 dseg->blk_ptr[i] &= ~3; 450 for (page_addr = dseg->blk_ptr[i]; 451 page_addr < dseg->blk_ptr[i] + dseg->blk_size; 452 page_addr += PAGE_SIZE) { 453 MEM_MAP_RESERVE(VIRT_TO_PAGE(page_addr)); 454 } 455 } else if (dseg->blk_ptr[i]) { 456 dseg->blk_ptr[i] &= ~3; 457 free_pages(dseg->blk_ptr[i], dseg->blk_order); 458 dseg->blk_ptr[i] = 0; 459 } 460 } 461 return dseg; 462 } 463 464 /* 465 * Function: _dma_segment_free 466 * 467 * Purpose: 468 * Release resources used by DMA segment. 469 * Parameters: 470 * dseg - DMA segment descriptor 471 * Returns: 472 * Nothing. 473 */ 474 static void 475 _dma_segment_free(dma_segment_t *dseg) 476 { 477 int i; 478 unsigned long page_addr; 479 480 if (dseg->blk_ptr) { 481 for (i = 0; i < dseg->blk_cnt; i++) { 482 if (dseg->blk_ptr[i]) { 483 for (page_addr = dseg->blk_ptr[i]; 484 page_addr < dseg->blk_ptr[i] + dseg->blk_size; 485 page_addr += PAGE_SIZE) { 486 MEM_MAP_UNRESERVE(VIRT_TO_PAGE(page_addr)); 487 } 488 free_pages(dseg->blk_ptr[i], dseg->blk_order); 489 } 490 } 491 kfree(dseg->blk_ptr); 492 kfree(dseg); 493 } 494 } 495 496 /* 497 * Function: _pgalloc 498 * 499 * Purpose: 500 * Allocate DMA memory using page allocator 501 * Parameters: 502 * size - number of bytes to allocate 503 * Returns: 504 * Pointer to allocated DMA memory or NULL if failure. 505 * Notes: 506 * For any sizes less than DMA_BLOCK_SIZE, we ask the page 507 * allocator for the entire memory block, otherwise we try 508 * to assemble a contiguous segment ourselves. 509 */ 510 static void * 511 _pgalloc(size_t size) 512 { 513 dma_segment_t *dseg; 514 size_t blk_size; 515 516 blk_size = (size < DMA_BLOCK_SIZE) ? size : DMA_BLOCK_SIZE; 517 if ((dseg = _dma_segment_alloc(size, blk_size)) == NULL) { 518 return NULL; 519 } 520 if (dseg->seg_size < size) { 521 /* If we didn't get the full size then forget it */ 522 gprintk("_pgalloc() failed to get requested size %zu: " 523 "only got %lu contiguous across %d blocks\n", 524 size, dseg->seg_size, dseg->blk_cnt); 525 _dma_segment_free(dseg); 526 return NULL; 527 } 528 list_add(&dseg->list, &_dma_seg); 529 return (void *)dseg->seg_begin; 530 } 531 532 /* 533 * Function: _pgfree 534 * 535 * Purpose: 536 * Free memory allocated by _pgalloc 537 * Parameters: 538 * ptr - pointer returned by _pgalloc 539 * Returns: 540 * 0 if succesfully freed, otherwise -1. 541 */ 542 static int 543 _pgfree(void *ptr) 544 { 545 struct list_head *pos; 546 list_for_each(pos, &_dma_seg) { 547 dma_segment_t *dseg = list_entry(pos, dma_segment_t, list); 548 if (ptr == (void *)dseg->seg_begin) { 549 list_del(&dseg->list); 550 _dma_segment_free(dseg); 551 return 0; 552 } 553 } 554 return -1; 555 } 556 557 /* 558 * Function: _pgcleanup 559 * 560 * Purpose: 561 * Free all memory allocated by _pgalloc 562 * Parameters: 563 * None 564 * Returns: 565 * Nothing. 566 */ 567 static void 568 _pgcleanup(void) 569 { 570 switch (dmaalloc) { 571 #if _SIMPLE_MEMORY_ALLOCATION_ 572 case ALLOC_TYPE_API: 573 if (_dma_vbase) { 574 if (dma_debug >= 1) gprintk("freeing v=%p p=0x%lx size=0x%lx\n", _dma_vbase,(unsigned long) _dma_pbase, (unsigned long)_dma_mem_size); 575 dma_free_coherent(DMA_DEV(DMA_DEV_INDEX), _dma_mem_size, _dma_vbase, _dma_pbase); 576 } 577 break; 578 #endif /* _SIMPLE_MEMORY_ALLOCATION_ */ 579 580 case ALLOC_TYPE_CHUNK: { 581 struct list_head *pos, *tmp; 582 int i, ndevices; 583 if (_use_dma_mapping) { 584 ndevices = BDE_NUM_DEVICES(BDE_SWITCH_DEVICES); 585 for (i = 0; i < ndevices && DMA_DEV(i); i ++) { 586 dma_unmap_single(DMA_DEV(i), (dma_addr_t)_dma_pbase, _dma_mem_size, DMA_BIDIRECTIONAL); 587 } 588 _use_dma_mapping = 0; 589 } 590 list_for_each_safe(pos, tmp, &_dma_seg) { 591 dma_segment_t *dseg = list_entry(pos, dma_segment_t, list); 592 list_del(&dseg->list); 593 _dma_segment_free(dseg); 594 } 595 break; 596 } 597 598 default: 599 gprintk("DMA memory allocation method dmaalloc=%d is not supported\n", dmaalloc); 600 } 601 } 602 603 /* 604 * Function: _alloc_mpool 605 * 606 * Purpose: 607 * Allocate DMA memory pool 608 * Parameters: 609 * size - size of DMA memory pool 610 * Returns: 611 * Nothing. 612 * Notes: 613 * If set up to use high memory, we simply map the memory into 614 * kernel space. 615 * It is assumed there is only one pool. 616 */ 617 static void 618 _alloc_mpool(size_t size) 619 { 620 unsigned long pbase = 0; 621 struct device *dev = DMA_DEV(DMA_DEV_INDEX); 622 int dma64_support = 0; 623 624 #if defined(IPROC_CMICD) && defined(CONFIG_OF) 625 if (of_find_compatible_node(NULL, NULL, "brcm,iproc-cmicx")) { 626 dma64_support = 1; 627 } 628 #endif 629 630 #if defined(__arm__) && !defined(CONFIG_HIGHMEM) 631 if (_use_himem) { 632 gprintk("DMA in high memory requires CONFIG_HIGHMEM on ARM CPUs.\n"); 633 return; 634 } 635 #endif 636 637 if (_use_himem) { 638 /* Use high memory for DMA */ 639 if (_himemaddr) { 640 pbase = _himemaddr; 641 } else { 642 pbase = virt_to_bus(high_memory); 643 } 644 if (((pbase + (size - 1)) >> 16) > DMA_BIT_MASK(16)) { 645 gprintk("DMA in high memory at 0x%lx size 0x%lx is beyond the 4GB limit and not supported.\n", pbase, (unsigned long)size); 646 return; 647 } 648 _cpu_pbase = _dma_pbase = pbase; 649 _dma_vbase = IOREMAP(_dma_pbase, size); 650 } else { 651 /* Get DMA memory from kernel */ 652 if (dma_debug >= 1) { 653 gprintk("Allocating DMA memory using method dmaalloc=%d\n", dmaalloc); 654 } 655 switch (dmaalloc) { 656 #if _SIMPLE_MEMORY_ALLOCATION_ 657 case ALLOC_TYPE_API: { 658 size_t alloc_size = size; /* size of memory allocated in current iteration */ 659 if (alloc_size > DMA_MAX_ALLOC_SIZE) { 660 alloc_size = DMA_MAX_ALLOC_SIZE; 661 } 662 /* get a memory allocation from the kernel */ 663 { 664 dma_addr_t dma_handle; 665 _dma_vbase = dma_alloc_coherent(dev, alloc_size, &dma_handle, GFP_KERNEL); 666 if (!_dma_vbase || !dma_handle) { 667 gprintk("Failed to allocate coherent memory pool of size 0x%lx\n", (unsigned long)alloc_size); 668 return; 669 } 670 _cpu_pbase = pbase = dma_handle; 671 } 672 673 if (alloc_size != size) { 674 gprintk("allocated 0x%lx bytes instead of 0x%lx bytes.\n", 675 (unsigned long)alloc_size, (unsigned long)size); 676 } 677 size = _dma_mem_size = alloc_size; 678 break; 679 } 680 #endif /* _SIMPLE_MEMORY_ALLOCATION_ */ 681 682 case ALLOC_TYPE_CHUNK: 683 _dma_vbase = _pgalloc(size); 684 if (!_dma_vbase) { 685 gprintk("Failed to allocate memory pool of size 0x%lx\n", (unsigned long)size); 686 return; 687 } 688 _cpu_pbase = virt_to_bus(_dma_vbase); 689 /* Use dma_map_single to obtain DMA bus address or IOVA if iommu is present. */ 690 if (dev) { 691 pbase = dma_map_single(dev, _dma_vbase, size, DMA_BIDIRECTIONAL); 692 if (BDE_DMA_MAPPING_ERROR(dev, pbase)) { 693 gprintk("Failed to map memory at %p\n", _dma_vbase); 694 _pgcleanup(); 695 _dma_vbase = NULL; 696 return; 697 } 698 _use_dma_mapping = 1; 699 } else { 700 pbase = _cpu_pbase; 701 } 702 break; 703 default: 704 _dma_vbase = NULL; 705 gprintk("DMA memory allocation method dmaalloc=%d is not supported\n", dmaalloc); 706 return; 707 } 708 709 _dma_pbase = pbase; 710 711 if (!dma64_support && ((pbase + (size - 1)) >> 16) > DMA_BIT_MASK(16)) { 712 gprintk("DMA memory allocated at 0x%lx size 0x%lx is beyond the 4GB limit and not supported.\n", pbase, (unsigned long)size); 713 _pgcleanup(); 714 _dma_vbase = NULL; 715 _dma_pbase = 0; 716 return; 717 } 718 719 #ifdef REMAP_DMA_NONCACHED 720 _dma_vbase = IOREMAP(_dma_pbase, size); 721 #endif 722 if (dma_debug >= 1) { 723 gprintk("_use_dma_mapping:%d _dma_vbase:%p _dma_pbase:%lx _cpu_pbase:%lx allocated:%lx dmaalloc:%d, dma64_support:%d\n", 724 _use_dma_mapping, _dma_vbase, (unsigned long)_dma_pbase, 725 (unsigned long)_cpu_pbase, (unsigned long)size, dmaalloc, dma64_support); 726 } 727 } 728 } 729 730 /* 731 * Function: _dma_cleanup 732 * 733 * Purpose: 734 * DMA cleanup function. 735 * Parameters: 736 * None 737 * Returns: 738 * Always 0 739 */ 740 int 741 _dma_cleanup(void) 742 { 743 if (_dma_vbase) { 744 mpool_destroy(_dma_pool); 745 if (_use_himem) { 746 iounmap(_dma_vbase); 747 } else { 748 #ifdef REMAP_DMA_NONCACHED 749 iounmap(_dma_vbase); 750 #endif 751 _pgcleanup(); 752 } 753 _dma_vbase = NULL; 754 _dma_pbase = 0; 755 _cpu_pbase = 0; 756 } 757 return 0; 758 } 759 760 void _dma_init(int dev_index) 761 { 762 unsigned long pbase; 763 764 if (dev_index > DMA_DEV_INDEX) { 765 if (_use_dma_mapping && DMA_DEV(dev_index) && _dma_vbase) { 766 pbase = dma_map_single(DMA_DEV(dev_index), _dma_vbase, _dma_mem_size, DMA_BIDIRECTIONAL); 767 if (BDE_DMA_MAPPING_ERROR(DMA_DEV(dev_index), pbase)) { 768 gprintk("Failed to map memory for device %d at %p\n", dev_index, _dma_vbase); 769 return; 770 } 771 if (pbase != (unsigned long)_dma_pbase) { 772 /* Bus address/IOVA must be identical for all devices. */ 773 gprintk("Device %d has different pbase: %lx (should be %lx)\n", 774 dev_index, pbase, (unsigned long)_dma_pbase); 775 } 776 } 777 return; 778 } 779 780 /* DMA Setup */ 781 if (dmasize) { 782 if ((dmasize[strlen(dmasize)-1] & ~0x20) == 'M') { 783 _dma_mem_size = simple_strtoul(dmasize, NULL, 0); 784 _dma_mem_size *= ONE_MB; 785 } else { 786 gprintk("DMA memory size must be specified as e.g. dmasize=8M\n"); 787 } 788 if (_dma_mem_size & (_dma_mem_size-1)) { 789 gprintk("dmasize must be a power of 2 (1M, 2M, 4M, 8M etc.)\n"); 790 _dma_mem_size = 0; 791 } 792 } 793 794 if (himem) { 795 if ((himem[0] & ~0x20) == 'Y' || himem[0] == '1') { 796 _use_himem = 1; 797 } else if ((himem[0] & ~0x20) == 'N' || himem[0] == '0') { 798 _use_himem = 0; 799 } 800 } 801 802 if (himemaddr && strlen(himemaddr) > 0) { 803 char suffix = (himemaddr[strlen(himemaddr)-1] & ~0x20); 804 _himemaddr = simple_strtoul(himemaddr, NULL, 0); 805 if (suffix == 'M') { 806 _himemaddr *= ONE_MB; 807 } else if (suffix == 'G') { 808 _himemaddr *= ONE_GB; 809 } else { 810 gprintk("DMA high memory address must be specified as e.g. himemaddr=8[MG]\n"); 811 } 812 } 813 814 if (_dma_mem_size) { 815 _alloc_mpool(_dma_mem_size); 816 if (_dma_vbase == NULL) { 817 gprintk("no DMA memory available\n"); 818 } else { 819 mpool_init(); 820 _dma_pool = mpool_create(_dma_vbase, _dma_mem_size); 821 } 822 } 823 } 824 825 /* 826 * Some kernels are configured to prevent mapping of kernel RAM memory 827 * into user space via the /dev/mem device. 828 * 829 * The function below provides a backdoor to mapping the DMA pool to 830 * user space via the BDE device file. 831 */ 832 int _dma_mmap(struct file *filp, struct vm_area_struct *vma) 833 { 834 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT; 835 unsigned long size = vma->vm_end - vma->vm_start; 836 837 if (phys_addr < (unsigned long )_cpu_pbase || 838 (phys_addr + size) > ((unsigned long )_cpu_pbase + _dma_mem_size)) { 839 gprintk("range 0x%lx-0x%lx outside DMA pool 0x%lx-0x%lx\n", 840 phys_addr, phys_addr + size, (unsigned long )_cpu_pbase, 841 (unsigned long )_cpu_pbase + _dma_mem_size); 842 return -EINVAL; 843 } 844 845 #ifdef USE_DMA_MMAP_COHERENT 846 if (dmaalloc == ALLOC_TYPE_API) { 847 vma->vm_pgoff = 0; 848 return dma_mmap_coherent(DMA_DEV(DMA_DEV_INDEX), vma, (void *)_dma_vbase, phys_addr, size); 849 } 850 #endif 851 852 _PGPROT_NONCACHED(vma->vm_page_prot); 853 854 if (remap_pfn_range(vma, 855 vma->vm_start, 856 vma->vm_pgoff, 857 size, 858 vma->vm_page_prot)) { 859 gprintk("Failed to mmap phys range 0x%lx-0x%lx to 0x%lx-0x%lx\n", 860 phys_addr, phys_addr + size, vma->vm_start,vma->vm_end); 861 return -EAGAIN; 862 } 863 return 0; 864 } 865 866 /* 867 * Function: _dma_pool_allocated 868 * 869 * Purpose: 870 * Check if DMA pool has been allocated. 871 * Parameters: 872 * None 873 * Returns: 874 * 0 : not allocated 875 * 1 : allocated 876 */ 877 int 878 _dma_pool_allocated(void) 879 { 880 return (_dma_vbase) ? 1 : 0; 881 } 882 883 sal_paddr_t 884 _l2p(int d, void *vaddr) 885 { 886 if (_dma_mem_size) { 887 /* dma memory is a contiguous block */ 888 if (vaddr) { 889 return _dma_pbase + (PTR_TO_UINTPTR(vaddr) - PTR_TO_UINTPTR(_dma_vbase)); 890 } 891 return 0; 892 } 893 return ((sal_paddr_t)virt_to_bus(vaddr)); 894 } 895 896 void * 897 _p2l(int d, sal_paddr_t paddr) 898 { 899 sal_vaddr_t vaddr = (sal_vaddr_t)_dma_vbase; 900 901 if (_dma_mem_size) { 902 /* DMA memory is a contiguous block */ 903 if (paddr == 0) { 904 return NULL; 905 } 906 return (void *)(vaddr + (sal_vaddr_t)(paddr - _dma_pbase)); 907 } 908 return bus_to_virt(paddr); 909 } 910 911 /* 912 * Some of the driver malloc's are too large for 913 * kmalloc(), so 'sal_alloc' and 'sal_free' in the 914 * linux kernel sal cannot be implemented with kmalloc(). 915 * 916 * Instead, they expect someone to provide an allocator 917 * that can handle the gimongous size of some of the 918 * allocations, and we provide it here, by allocating 919 * this memory out of the boot-time dma pool. 920 * 921 * These are the functions in question: 922 */ 923 924 void* kmalloc_giant(int sz) 925 { 926 return mpool_alloc(_dma_pool, sz); 927 } 928 929 void kfree_giant(void* ptr) 930 { 931 return mpool_free(_dma_pool, ptr); 932 } 933 934 uint32_t * 935 _salloc(int d, int size, const char *name) 936 { 937 void *ptr; 938 939 if (_dma_mem_size) { 940 return mpool_alloc(_dma_pool, size); 941 } 942 if ((ptr = kmalloc(size, mem_flags)) == NULL) { 943 ptr = _pgalloc(size); 944 } 945 return ptr; 946 } 947 948 void 949 _sfree(int d, void *ptr) 950 { 951 if (_dma_mem_size) { 952 return mpool_free(_dma_pool, ptr); 953 } 954 if (_pgfree(ptr) < 0) { 955 kfree(ptr); 956 } 957 } 958 959 int 960 _sinval(int d, void *ptr, int length) 961 { 962 #if defined(dma_cache_wback_inv) 963 dma_cache_wback_inv((unsigned long)ptr, length); 964 #else 965 #if defined(IPROC_CMICD) || defined(BCM958525) 966 /* FIXME: need proper function to replace dma_cache_sync */ 967 dma_sync_single_for_cpu(NULL, (unsigned long)ptr, length, DMA_BIDIRECTIONAL); 968 #else 969 dma_cache_sync(NULL, ptr, length, DMA_BIDIRECTIONAL); 970 #endif 971 #endif 972 return 0; 973 } 974 975 int 976 _sflush(int d, void *ptr, int length) 977 { 978 #if defined(dma_cache_wback_inv) 979 dma_cache_wback_inv((unsigned long)ptr, length); 980 #else 981 #if defined(IPROC_CMICD) || defined(BCM958525) 982 /* FIXME: need proper function to replace dma_cache_sync */ 983 dma_sync_single_for_cpu(NULL, (unsigned long)ptr, length, DMA_BIDIRECTIONAL); 984 #else 985 dma_cache_sync(NULL, ptr, length, DMA_BIDIRECTIONAL); 986 #endif 987 #endif 988 989 return 0; 990 } 991 992 int 993 lkbde_get_dma_info(phys_addr_t* cpu_pbase, phys_addr_t* dma_pbase, ssize_t* size) 994 { 995 if (_dma_vbase == NULL) { 996 if (_dma_mem_size == 0) { 997 _dma_mem_size = DMA_MEM_DEFAULT; 998 } 999 _alloc_mpool(_dma_mem_size); 1000 } 1001 *cpu_pbase = _cpu_pbase; 1002 *dma_pbase = _dma_pbase; 1003 *size = (_dma_vbase) ? _dma_mem_size : 0; 1004 return 0; 1005 } 1006 1007 void 1008 _dma_pprint(void) 1009 { 1010 pprintf("\tdmasize=%s\n", dmasize); 1011 pprintf("\thimem=%s\n", himem); 1012 pprintf("\thimemaddr=%s\n", himemaddr); 1013 pprintf("DMA Memory (%s): %d bytes, %d used, %d free%s\n", 1014 (_use_himem) ? "high" : "kernel", 1015 (_dma_vbase) ? _dma_mem_size : 0, 1016 (_dma_vbase) ? mpool_usage(_dma_pool) : 0, 1017 (_dma_vbase) ? _dma_mem_size - mpool_usage(_dma_pool) : 0, 1018 USE_LINUX_BDE_MMAP ? ", local mmap" : ""); 1019 } 1020 1021 /* 1022 * Export functions 1023 */ 1024 LKM_EXPORT_SYM(kmalloc_giant); 1025 LKM_EXPORT_SYM(kfree_giant); 1026 LKM_EXPORT_SYM(lkbde_get_dma_info);